

<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
  <meta charset="utf-8" />
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  
  <title>mindspore.dataset.engine.datasets_vision &mdash; MindSpore master documentation</title>
  

  
  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />

  
  

  
  

  

  
  <!--[if lt IE 9]>
    <script src="../../../../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
        <script src="../../../../_static/jquery.js"></script>
        <script src="../../../../_static/underscore.js"></script>
        <script src="../../../../_static/doctools.js"></script>
        <script src="../../../../_static/language_data.js"></script>
        <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
    
    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>

    
    <link rel="index" title="Index" href="../../../../genindex.html" />
    <link rel="search" title="Search" href="../../../../search.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../../../../index.html" class="icon icon-home"> MindSpore
          

          
          </a>

          
            
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">MindSpore Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.html">mindspore</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.common.initializer.html">mindspore.common.initializer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.communication.html">mindspore.communication</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.compression.html">mindspore.compression</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.context.html">mindspore.context</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.html">mindspore.dataset</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.audio.html">mindspore.dataset.audio</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.config.html">mindspore.dataset.config</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.text.html">mindspore.dataset.text</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.transforms.html">mindspore.dataset.transforms</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.dataset.vision.html">mindspore.dataset.vision</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.mindrecord.html">mindspore.mindrecord</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.nn.html">mindspore.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.nn.probability.html">mindspore.nn.probability</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.nn.transformer.html">mindspore.nn.transformer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.numpy.html">mindspore.numpy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.ops.html">mindspore.ops</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.parallel.html">mindspore.parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.parallel.nn.html">mindspore.parallel.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.profiler.html">mindspore.profiler</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.scipy.html">mindspore.scipy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.train.html">mindspore.train</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api_python/mindspore.boost.html">mindspore.boost</a></li>
</ul>
<p class="caption"><span class="caption-text">MindSpore C++ API</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://www.mindspore.cn/lite/api/zh-CN/master/api_cpp/mindspore.html">MindSpore Lite↗</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../../../../index.html">MindSpore</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          

















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../../../index.html" class="icon icon-home"></a> &raquo;</li>
        
          <li><a href="../../../index.html">Module code</a> &raquo;</li>
        
      <li>mindspore.dataset.engine.datasets_vision</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <h1>Source code for mindspore.dataset.engine.datasets_vision</h1><div class="highlight"><pre>
<span></span><span class="c1"># Copyright 2019-2022 Huawei Technologies Co., Ltd</span>
<span class="c1">#</span>
<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
<span class="c1"># you may not use this file except in compliance with the License.</span>
<span class="c1"># You may obtain a copy of the License at</span>
<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
<span class="c1"># ==============================================================================</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">This file contains specific vision dataset loading classes. You can easily use</span>
<span class="sd">these classes to load the prepared dataset. For example:</span>
<span class="sd">    ImageFolderDataset: which is about ImageNet dataset.</span>
<span class="sd">    Cifar10Dataset: which is cifar10 binary version dataset.</span>
<span class="sd">    Cifar100Dataset: which is cifar100 binary version dataset.</span>
<span class="sd">    MnistDataset: which is mnist dataset.</span>
<span class="sd">    ...</span>
<span class="sd">After declaring the dataset object, you can further apply dataset operations</span>
<span class="sd">(e.g. filter, skip, concat, map, batch) on it.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">from</span> <span class="nn">scipy.io</span> <span class="kn">import</span> <span class="n">loadmat</span>
<span class="kn">from</span> <span class="nn">PIL</span> <span class="kn">import</span> <span class="n">Image</span>

<span class="kn">import</span> <span class="nn">mindspore._c_dataengine</span> <span class="k">as</span> <span class="nn">cde</span>

<span class="kn">from</span> <span class="nn">.datasets</span> <span class="kn">import</span> <span class="n">VisionBaseDataset</span><span class="p">,</span> <span class="n">SourceDataset</span><span class="p">,</span> <span class="n">MappableDataset</span><span class="p">,</span> <span class="n">Shuffle</span><span class="p">,</span> <span class="n">Schema</span>
<span class="kn">from</span> <span class="nn">.datasets_user_defined</span> <span class="kn">import</span> <span class="n">GeneratorDataset</span>
<span class="kn">from</span> <span class="nn">.validators</span> <span class="kn">import</span> <span class="n">check_imagefolderdataset</span><span class="p">,</span> \
    <span class="n">check_mnist_cifar_dataset</span><span class="p">,</span> <span class="n">check_manifestdataset</span><span class="p">,</span> <span class="n">check_vocdataset</span><span class="p">,</span> <span class="n">check_cocodataset</span><span class="p">,</span> \
    <span class="n">check_celebadataset</span><span class="p">,</span> <span class="n">check_flickr_dataset</span><span class="p">,</span> <span class="n">check_sb_dataset</span><span class="p">,</span> <span class="n">check_flowers102dataset</span><span class="p">,</span> <span class="n">check_cityscapes_dataset</span><span class="p">,</span> \
    <span class="n">check_usps_dataset</span><span class="p">,</span> <span class="n">check_div2k_dataset</span><span class="p">,</span> <span class="n">check_random_dataset</span><span class="p">,</span> \
    <span class="n">check_sbu_dataset</span><span class="p">,</span> <span class="n">check_qmnist_dataset</span><span class="p">,</span> <span class="n">check_emnist_dataset</span><span class="p">,</span> <span class="n">check_fake_image_dataset</span><span class="p">,</span> <span class="n">check_places365_dataset</span><span class="p">,</span> \
    <span class="n">check_photo_tour_dataset</span><span class="p">,</span> <span class="n">check_svhn_dataset</span><span class="p">,</span> <span class="n">check_stl10_dataset</span><span class="p">,</span> <span class="n">check_semeion_dataset</span><span class="p">,</span> \
    <span class="n">check_caltech101_dataset</span><span class="p">,</span> <span class="n">check_caltech256_dataset</span><span class="p">,</span> <span class="n">check_wider_face_dataset</span>

<span class="kn">from</span> <span class="nn">..core.validator_helpers</span> <span class="kn">import</span> <span class="n">replace_none</span>


<span class="k">class</span> <span class="nc">_Caltech101Dataset</span><span class="p">:</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Mainly for loading Caltech101 Dataset, and return two rows each time.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">target_type</span><span class="o">=</span><span class="s2">&quot;category&quot;</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">realpath</span><span class="p">(</span><span class="n">dataset_dir</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">image_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="s2">&quot;101_ObjectCategories&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">annotation_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="s2">&quot;Annotations&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">target_type</span> <span class="o">=</span> <span class="n">target_type</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_type</span> <span class="o">==</span> <span class="s2">&quot;category&quot;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">column_names</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">,</span> <span class="s2">&quot;category&quot;</span><span class="p">]</span>
        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_type</span> <span class="o">==</span> <span class="s2">&quot;annotation&quot;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">column_names</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">,</span> <span class="s2">&quot;annotation&quot;</span><span class="p">]</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">column_names</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">,</span> <span class="s2">&quot;category&quot;</span><span class="p">,</span> <span class="s2">&quot;annotation&quot;</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">decode</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">classes</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">image_dir</span><span class="p">))</span>
        <span class="k">if</span> <span class="s2">&quot;BACKGROUND_Google&quot;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">classes</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">classes</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="s2">&quot;BACKGROUND_Google&quot;</span><span class="p">)</span>
        <span class="n">name_map</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;Faces&quot;</span><span class="p">:</span> <span class="s2">&quot;Faces_2&quot;</span><span class="p">,</span>
                    <span class="s2">&quot;Faces_easy&quot;</span><span class="p">:</span> <span class="s2">&quot;Faces_3&quot;</span><span class="p">,</span>
                    <span class="s2">&quot;Motorbikes&quot;</span><span class="p">:</span> <span class="s2">&quot;Motorbikes_16&quot;</span><span class="p">,</span>
                    <span class="s2">&quot;airplanes&quot;</span><span class="p">:</span> <span class="s2">&quot;Airplanes_Side_2&quot;</span><span class="p">}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">annotation_classes</span> <span class="o">=</span> <span class="p">[</span><span class="n">name_map</span><span class="p">[</span><span class="n">class_name</span><span class="p">]</span> <span class="k">if</span> <span class="n">class_name</span> <span class="ow">in</span> <span class="n">name_map</span> <span class="k">else</span> <span class="n">class_name</span>
                                   <span class="k">for</span> <span class="n">class_name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">classes</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">image_index</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">image_label</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">image_class</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">classes</span><span class="p">):</span>
            <span class="n">sub_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">image_dir</span><span class="p">,</span> <span class="n">image_class</span><span class="p">)</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">sub_dir</span><span class="p">)</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">access</span><span class="p">(</span><span class="n">sub_dir</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">R_OK</span><span class="p">):</span>
                <span class="k">continue</span>
            <span class="n">num_images</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">sub_dir</span><span class="p">))</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">image_index</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">num_images</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">image_label</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">num_images</span> <span class="o">*</span> <span class="p">[</span><span class="n">i</span><span class="p">])</span>

    <span class="k">def</span> <span class="fm">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">index</span><span class="p">):</span>
        <span class="n">image_file</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">image_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">classes</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">image_label</span><span class="p">[</span><span class="n">index</span><span class="p">]],</span>
                                  <span class="s2">&quot;image_</span><span class="si">{:04d}</span><span class="s2">.jpg&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">image_index</span><span class="p">[</span><span class="n">index</span><span class="p">]))</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">image_file</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;The image file </span><span class="si">{}</span><span class="s2"> does not exist or permission denied!&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">image_file</span><span class="p">))</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">:</span>
            <span class="n">image</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">Image</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">image_file</span><span class="p">)</span><span class="o">.</span><span class="n">convert</span><span class="p">(</span><span class="s2">&quot;RGB&quot;</span><span class="p">))</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">image</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">fromfile</span><span class="p">(</span><span class="n">image_file</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_type</span> <span class="o">==</span> <span class="s2">&quot;category&quot;</span><span class="p">:</span>
            <span class="k">return</span> <span class="n">image</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_label</span><span class="p">[</span><span class="n">index</span><span class="p">]</span>
        <span class="n">annotation_file</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">annotation_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">annotation_classes</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">image_label</span><span class="p">[</span><span class="n">index</span><span class="p">]],</span>
                                       <span class="s2">&quot;annotation_</span><span class="si">{:04d}</span><span class="s2">.mat&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">image_index</span><span class="p">[</span><span class="n">index</span><span class="p">]))</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">annotation_file</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;The annotation file </span><span class="si">{}</span><span class="s2"> does not exist or permission denied!&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">annotation_file</span><span class="p">))</span>
        <span class="n">annotation</span> <span class="o">=</span> <span class="n">loadmat</span><span class="p">(</span><span class="n">annotation_file</span><span class="p">)[</span><span class="s2">&quot;obj_contour&quot;</span><span class="p">]</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_type</span> <span class="o">==</span> <span class="s2">&quot;annotation&quot;</span><span class="p">:</span>
            <span class="k">return</span> <span class="n">image</span><span class="p">,</span> <span class="n">annotation</span>
        <span class="k">return</span> <span class="n">image</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_label</span><span class="p">[</span><span class="n">index</span><span class="p">],</span> <span class="n">annotation</span>

    <span class="k">def</span> <span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">image_index</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">Caltech101Dataset</span><span class="p">(</span><span class="n">GeneratorDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses Caltech101 dataset.</span>

<span class="sd">    The columns of the generated dataset depend on the value of `target_type`.</span>
<span class="sd">    When `target_type` is `category`, the columns are :py:obj:`[image, category]`.</span>
<span class="sd">    When `target_type` is `annotation`, the columns are :py:obj:`[image, annotation]`.</span>
<span class="sd">    When `target_type` is `all`, the columns are :py:obj:`[image, category, annotation]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`category` is of the uint32 type.</span>
<span class="sd">    The tensor of column :py:obj:`annotation` is a 2-dimensional ndarray that stores the contour of the image</span>
<span class="sd">    and consists of a series of points.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset. This root directory contains two</span>
<span class="sd">            subdirectories, one is called 101_ObjectCategories, which stores images,</span>
<span class="sd">            and the other is called Annotations, which stores annotations.</span>
<span class="sd">        target_type (str, optional): Target of the image. If target_type is &quot;category&quot;, return category represents</span>
<span class="sd">            the target class. If target_type is &quot;annotation&quot;, return annotation.</span>
<span class="sd">            If target_type is &quot;all&quot;, return category and annotation (default=None, means &quot;category&quot;).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data (default=1).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        decode (bool, optional): Whether or not to decode the images after reading (default=False).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If target_type is not set correctly.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; caltech101_dataset_directory = &quot;/path/to/caltech101_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Read all samples (image files) in caltech101_dataset_directory with 8 threads</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.Caltech101Dataset(dataset_dir=caltech101_dataset_directory, num_parallel_workers=8)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Read all samples (image files) with the target_type &quot;annotation&quot;</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.Caltech101Dataset(dataset_dir=caltech101_dataset_directory, target_type=&quot;annotation&quot;)</span>

<span class="sd">    About Caltech101Dataset:</span>

<span class="sd">    Pictures of objects belonging to 101 categories. About 40 to 800 images per category.</span>
<span class="sd">    Most categories have about 50 images. Collected in September 2003 by Fei-Fei Li, Marco Andreetto,</span>
<span class="sd">    and Marc &#39;Aurelio Ranzato. The size of each image is roughly 300 x 200 pixels.</span>
<span class="sd">    The official provides the contour data of each object in each picture, which is the annotation.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── caltech101_dataset_directory</span>
<span class="sd">            ├── 101_ObjectCategories</span>
<span class="sd">            │    ├── Faces</span>
<span class="sd">            │    │    ├── image_0001.jpg</span>
<span class="sd">            │    │    ├── image_0002.jpg</span>
<span class="sd">            │    │    ...</span>
<span class="sd">            │    ├── Faces_easy</span>
<span class="sd">            │    │    ├── image_0001.jpg</span>
<span class="sd">            │    │    ├── image_0002.jpg</span>
<span class="sd">            │    │    ...</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            └── Annotations</span>
<span class="sd">                 ├── Airplanes_Side_2</span>
<span class="sd">                 │    ├── annotation_0001.mat</span>
<span class="sd">                 │    ├── annotation_0002.mat</span>
<span class="sd">                 │    ...</span>
<span class="sd">                 ├── Faces_2</span>
<span class="sd">                 │    ├── annotation_0001.mat</span>
<span class="sd">                 │    ├── annotation_0002.mat</span>
<span class="sd">                 │    ...</span>
<span class="sd">                 ├── ...</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @article{FeiFei2004LearningGV,</span>
<span class="sd">        author    = {Li Fei-Fei and Rob Fergus and Pietro Perona},</span>
<span class="sd">        title     = {Learning Generative Visual Models from Few Training Examples:</span>
<span class="sd">                    An Incremental Bayesian Approach Tested on 101 Object Categories},</span>
<span class="sd">        journal   = {Computer Vision and Pattern Recognition Workshop},</span>
<span class="sd">        year      = {2004},</span>
<span class="sd">        url       = {http://www.vision.caltech.edu/Image_Datasets/Caltech101/},</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_caltech101_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">target_type</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
                 <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">target_type</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">target_type</span><span class="p">,</span> <span class="s2">&quot;category&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
        <span class="n">dataset</span> <span class="o">=</span> <span class="n">_Caltech101Dataset</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_type</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">)</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">column_names</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">column_names</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span>
                         <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">get_class_indexing</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Get the class index.</span>

<span class="sd">        Returns:</span>
<span class="sd">            dict, a str-to-int mapping from label name to index.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">class_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;Faces&#39;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> <span class="s1">&#39;Faces_easy&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">&#39;Leopards&#39;</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span> <span class="s1">&#39;Motorbikes&#39;</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> <span class="s1">&#39;accordion&#39;</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="s1">&#39;airplanes&#39;</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span>
                      <span class="s1">&#39;anchor&#39;</span><span class="p">:</span> <span class="mi">6</span><span class="p">,</span> <span class="s1">&#39;ant&#39;</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span> <span class="s1">&#39;barrel&#39;</span><span class="p">:</span> <span class="mi">8</span><span class="p">,</span> <span class="s1">&#39;bass&#39;</span><span class="p">:</span> <span class="mi">9</span><span class="p">,</span> <span class="s1">&#39;beaver&#39;</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span> <span class="s1">&#39;binocular&#39;</span><span class="p">:</span> <span class="mi">11</span><span class="p">,</span> <span class="s1">&#39;bonsai&#39;</span><span class="p">:</span> <span class="mi">12</span><span class="p">,</span>
                      <span class="s1">&#39;brain&#39;</span><span class="p">:</span> <span class="mi">13</span><span class="p">,</span> <span class="s1">&#39;brontosaurus&#39;</span><span class="p">:</span> <span class="mi">14</span><span class="p">,</span> <span class="s1">&#39;buddha&#39;</span><span class="p">:</span> <span class="mi">15</span><span class="p">,</span> <span class="s1">&#39;butterfly&#39;</span><span class="p">:</span> <span class="mi">16</span><span class="p">,</span> <span class="s1">&#39;camera&#39;</span><span class="p">:</span> <span class="mi">17</span><span class="p">,</span> <span class="s1">&#39;cannon&#39;</span><span class="p">:</span> <span class="mi">18</span><span class="p">,</span>
                      <span class="s1">&#39;car_side&#39;</span><span class="p">:</span> <span class="mi">19</span><span class="p">,</span> <span class="s1">&#39;ceiling_fan&#39;</span><span class="p">:</span> <span class="mi">20</span><span class="p">,</span> <span class="s1">&#39;cellphone&#39;</span><span class="p">:</span> <span class="mi">21</span><span class="p">,</span> <span class="s1">&#39;chair&#39;</span><span class="p">:</span> <span class="mi">22</span><span class="p">,</span> <span class="s1">&#39;chandelier&#39;</span><span class="p">:</span> <span class="mi">23</span><span class="p">,</span>
                      <span class="s1">&#39;cougar_body&#39;</span><span class="p">:</span> <span class="mi">24</span><span class="p">,</span> <span class="s1">&#39;cougar_face&#39;</span><span class="p">:</span> <span class="mi">25</span><span class="p">,</span> <span class="s1">&#39;crab&#39;</span><span class="p">:</span> <span class="mi">26</span><span class="p">,</span> <span class="s1">&#39;crayfish&#39;</span><span class="p">:</span> <span class="mi">27</span><span class="p">,</span> <span class="s1">&#39;crocodile&#39;</span><span class="p">:</span> <span class="mi">28</span><span class="p">,</span>
                      <span class="s1">&#39;crocodile_head&#39;</span><span class="p">:</span> <span class="mi">29</span><span class="p">,</span> <span class="s1">&#39;cup&#39;</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span> <span class="s1">&#39;dalmatian&#39;</span><span class="p">:</span> <span class="mi">31</span><span class="p">,</span> <span class="s1">&#39;dollar_bill&#39;</span><span class="p">:</span> <span class="mi">32</span><span class="p">,</span> <span class="s1">&#39;dolphin&#39;</span><span class="p">:</span> <span class="mi">33</span><span class="p">,</span>
                      <span class="s1">&#39;dragonfly&#39;</span><span class="p">:</span> <span class="mi">34</span><span class="p">,</span> <span class="s1">&#39;electric_guitar&#39;</span><span class="p">:</span> <span class="mi">35</span><span class="p">,</span> <span class="s1">&#39;elephant&#39;</span><span class="p">:</span> <span class="mi">36</span><span class="p">,</span> <span class="s1">&#39;emu&#39;</span><span class="p">:</span> <span class="mi">37</span><span class="p">,</span> <span class="s1">&#39;euphonium&#39;</span><span class="p">:</span> <span class="mi">38</span><span class="p">,</span> <span class="s1">&#39;ewer&#39;</span><span class="p">:</span> <span class="mi">39</span><span class="p">,</span>
                      <span class="s1">&#39;ferry&#39;</span><span class="p">:</span> <span class="mi">40</span><span class="p">,</span> <span class="s1">&#39;flamingo&#39;</span><span class="p">:</span> <span class="mi">41</span><span class="p">,</span> <span class="s1">&#39;flamingo_head&#39;</span><span class="p">:</span> <span class="mi">42</span><span class="p">,</span> <span class="s1">&#39;garfield&#39;</span><span class="p">:</span> <span class="mi">43</span><span class="p">,</span> <span class="s1">&#39;gerenuk&#39;</span><span class="p">:</span> <span class="mi">44</span><span class="p">,</span> <span class="s1">&#39;gramophone&#39;</span><span class="p">:</span> <span class="mi">45</span><span class="p">,</span>
                      <span class="s1">&#39;grand_piano&#39;</span><span class="p">:</span> <span class="mi">46</span><span class="p">,</span> <span class="s1">&#39;hawksbill&#39;</span><span class="p">:</span> <span class="mi">47</span><span class="p">,</span> <span class="s1">&#39;headphone&#39;</span><span class="p">:</span> <span class="mi">48</span><span class="p">,</span> <span class="s1">&#39;hedgehog&#39;</span><span class="p">:</span> <span class="mi">49</span><span class="p">,</span> <span class="s1">&#39;helicopter&#39;</span><span class="p">:</span> <span class="mi">50</span><span class="p">,</span> <span class="s1">&#39;ibis&#39;</span><span class="p">:</span> <span class="mi">51</span><span class="p">,</span>
                      <span class="s1">&#39;inline_skate&#39;</span><span class="p">:</span> <span class="mi">52</span><span class="p">,</span> <span class="s1">&#39;joshua_tree&#39;</span><span class="p">:</span> <span class="mi">53</span><span class="p">,</span> <span class="s1">&#39;kangaroo&#39;</span><span class="p">:</span> <span class="mi">54</span><span class="p">,</span> <span class="s1">&#39;ketch&#39;</span><span class="p">:</span> <span class="mi">55</span><span class="p">,</span> <span class="s1">&#39;lamp&#39;</span><span class="p">:</span> <span class="mi">56</span><span class="p">,</span> <span class="s1">&#39;laptop&#39;</span><span class="p">:</span> <span class="mi">57</span><span class="p">,</span>
                      <span class="s1">&#39;llama&#39;</span><span class="p">:</span> <span class="mi">58</span><span class="p">,</span> <span class="s1">&#39;lobster&#39;</span><span class="p">:</span> <span class="mi">59</span><span class="p">,</span> <span class="s1">&#39;lotus&#39;</span><span class="p">:</span> <span class="mi">60</span><span class="p">,</span> <span class="s1">&#39;mandolin&#39;</span><span class="p">:</span> <span class="mi">61</span><span class="p">,</span> <span class="s1">&#39;mayfly&#39;</span><span class="p">:</span> <span class="mi">62</span><span class="p">,</span> <span class="s1">&#39;menorah&#39;</span><span class="p">:</span> <span class="mi">63</span><span class="p">,</span>
                      <span class="s1">&#39;metronome&#39;</span><span class="p">:</span> <span class="mi">64</span><span class="p">,</span> <span class="s1">&#39;minaret&#39;</span><span class="p">:</span> <span class="mi">65</span><span class="p">,</span> <span class="s1">&#39;nautilus&#39;</span><span class="p">:</span> <span class="mi">66</span><span class="p">,</span> <span class="s1">&#39;octopus&#39;</span><span class="p">:</span> <span class="mi">67</span><span class="p">,</span> <span class="s1">&#39;okapi&#39;</span><span class="p">:</span> <span class="mi">68</span><span class="p">,</span> <span class="s1">&#39;pagoda&#39;</span><span class="p">:</span> <span class="mi">69</span><span class="p">,</span>
                      <span class="s1">&#39;panda&#39;</span><span class="p">:</span> <span class="mi">70</span><span class="p">,</span> <span class="s1">&#39;pigeon&#39;</span><span class="p">:</span> <span class="mi">71</span><span class="p">,</span> <span class="s1">&#39;pizza&#39;</span><span class="p">:</span> <span class="mi">72</span><span class="p">,</span> <span class="s1">&#39;platypus&#39;</span><span class="p">:</span> <span class="mi">73</span><span class="p">,</span> <span class="s1">&#39;pyramid&#39;</span><span class="p">:</span> <span class="mi">74</span><span class="p">,</span> <span class="s1">&#39;revolver&#39;</span><span class="p">:</span> <span class="mi">75</span><span class="p">,</span>
                      <span class="s1">&#39;rhino&#39;</span><span class="p">:</span> <span class="mi">76</span><span class="p">,</span> <span class="s1">&#39;rooster&#39;</span><span class="p">:</span> <span class="mi">77</span><span class="p">,</span> <span class="s1">&#39;saxophone&#39;</span><span class="p">:</span> <span class="mi">78</span><span class="p">,</span> <span class="s1">&#39;schooner&#39;</span><span class="p">:</span> <span class="mi">79</span><span class="p">,</span> <span class="s1">&#39;scissors&#39;</span><span class="p">:</span> <span class="mi">80</span><span class="p">,</span> <span class="s1">&#39;scorpion&#39;</span><span class="p">:</span> <span class="mi">81</span><span class="p">,</span>
                      <span class="s1">&#39;sea_horse&#39;</span><span class="p">:</span> <span class="mi">82</span><span class="p">,</span> <span class="s1">&#39;snoopy&#39;</span><span class="p">:</span> <span class="mi">83</span><span class="p">,</span> <span class="s1">&#39;soccer_ball&#39;</span><span class="p">:</span> <span class="mi">84</span><span class="p">,</span> <span class="s1">&#39;stapler&#39;</span><span class="p">:</span> <span class="mi">85</span><span class="p">,</span> <span class="s1">&#39;starfish&#39;</span><span class="p">:</span> <span class="mi">86</span><span class="p">,</span>
                      <span class="s1">&#39;stegosaurus&#39;</span><span class="p">:</span> <span class="mi">87</span><span class="p">,</span> <span class="s1">&#39;stop_sign&#39;</span><span class="p">:</span> <span class="mi">88</span><span class="p">,</span> <span class="s1">&#39;strawberry&#39;</span><span class="p">:</span> <span class="mi">89</span><span class="p">,</span> <span class="s1">&#39;sunflower&#39;</span><span class="p">:</span> <span class="mi">90</span><span class="p">,</span> <span class="s1">&#39;tick&#39;</span><span class="p">:</span> <span class="mi">91</span><span class="p">,</span>
                      <span class="s1">&#39;trilobite&#39;</span><span class="p">:</span> <span class="mi">92</span><span class="p">,</span> <span class="s1">&#39;umbrella&#39;</span><span class="p">:</span> <span class="mi">93</span><span class="p">,</span> <span class="s1">&#39;watch&#39;</span><span class="p">:</span> <span class="mi">94</span><span class="p">,</span> <span class="s1">&#39;water_lilly&#39;</span><span class="p">:</span> <span class="mi">95</span><span class="p">,</span> <span class="s1">&#39;wheelchair&#39;</span><span class="p">:</span> <span class="mi">96</span><span class="p">,</span> <span class="s1">&#39;wild_cat&#39;</span><span class="p">:</span> <span class="mi">97</span><span class="p">,</span>
                      <span class="s1">&#39;windsor_chair&#39;</span><span class="p">:</span> <span class="mi">98</span><span class="p">,</span> <span class="s1">&#39;wrench&#39;</span><span class="p">:</span> <span class="mi">99</span><span class="p">,</span> <span class="s1">&#39;yin_yang&#39;</span><span class="p">:</span> <span class="mi">100</span><span class="p">}</span>
        <span class="k">return</span> <span class="n">class_dict</span>


<span class="k">class</span> <span class="nc">Caltech256Dataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses Caltech256 dataset.</span>

<span class="sd">    The generated dataset has two columns: :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        decode (bool, optional): Whether or not to decode the images after reading (default=False).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; caltech256_dataset_dir = &quot;/path/to/caltech256_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Read all samples (image files) in caltech256_dataset_dir with 8 threads</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.Caltech256Dataset(dataset_dir=caltech256_dataset_dir, num_parallel_workers=8)</span>

<span class="sd">    About Caltech256Dataset:</span>

<span class="sd">    Caltech-256 is an object recognition dataset containing 30,607 real-world images, of different sizes,</span>
<span class="sd">    spanning 257 classes (256 object classes and an additional clutter class).</span>
<span class="sd">    Each class is represented by at least 80 images. The dataset is a superset of the Caltech-101 dataset.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── caltech256_dataset_directory</span>
<span class="sd">             ├── 001.ak47</span>
<span class="sd">             │    ├── 001_0001.jpg</span>
<span class="sd">             │    ├── 001_0002.jpg</span>
<span class="sd">             │    ...</span>
<span class="sd">             ├── 002.american-flag</span>
<span class="sd">             │    ├── 002_0001.jpg</span>
<span class="sd">             │    ├── 002_0002.jpg</span>
<span class="sd">             │    ...</span>
<span class="sd">             ├── 003.backpack</span>
<span class="sd">             │    ├── 003_0001.jpg</span>
<span class="sd">             │    ├── 003_0002.jpg</span>
<span class="sd">             │    ...</span>
<span class="sd">             ├── ...</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @article{griffin2007caltech,</span>
<span class="sd">        title     = {Caltech-256 object category dataset},</span>
<span class="sd">        added-at  = {2021-01-21T02:54:42.000+0100},</span>
<span class="sd">        author    = {Griffin, Gregory and Holub, Alex and Perona, Pietro},</span>
<span class="sd">        biburl    = {https://www.bibsonomy.org/bibtex/21f746f23ff0307826cca3e3be45f8de7/s364315},</span>
<span class="sd">        interhash = {bfe1e648c1778c04baa60f23d1223375},</span>
<span class="sd">        intrahash = {1f746f23ff0307826cca3e3be45f8de7},</span>
<span class="sd">        publisher = {California Institute of Technology},</span>
<span class="sd">        timestamp = {2021-01-21T02:54:42.000+0100},</span>
<span class="sd">        year      = {2007}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_caltech256_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
                 <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">Caltech256Node</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">CelebADataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses CelebA dataset.</span>
<span class="sd">    Only support to read `list_attr_celeba.txt` currently, which is the attribute annotations of the dataset.</span>

<span class="sd">    The generated dataset has two columns: :py:obj:`[image, attr]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`attr` is of the uint32 type and one hot encoded.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data (default=None, will use value set in</span>
<span class="sd">            the config).</span>
<span class="sd">        shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None).</span>
<span class="sd">        usage (str, optional): Specify the `train`, `valid`, `test` part or `all` parts of dataset</span>
<span class="sd">            (default= `all`, will read all samples).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the dataset (default=None).</span>
<span class="sd">        decode (bool, optional): decode the images after reading (default=False).</span>
<span class="sd">        extensions (list[str], optional): List of file extensions to be included in the dataset (default=None).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, will include all images).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This</span>
<span class="sd">            argument can only be specified when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; celeba_dataset_dir = &quot;/path/to/celeba_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Read 5 samples from CelebA dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.CelebADataset(dataset_dir=celeba_dataset_dir, usage=&#39;train&#39;, num_samples=5)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Note: In celeba dataset, each data dictionary owns keys &quot;image&quot; and &quot;attr&quot;</span>

<span class="sd">    About CelebA dataset:</span>

<span class="sd">    CelebFaces Attributes Dataset (CelebA) is a large-scale face attributes dataset</span>
<span class="sd">    with more than 200K celebrity images, each with 40 attribute annotations.</span>

<span class="sd">    The images in this dataset cover large pose variations and background clutter.</span>
<span class="sd">    CelebA has large diversities, large quantities, and rich annotations, including</span>

<span class="sd">    * 10,177 number of identities,</span>
<span class="sd">    * 202,599 number of face images,</span>
<span class="sd">    * 5 landmark locations, 40 binary attributes annotations per image.</span>

<span class="sd">    The dataset can be employed as the training and test sets for the following computer</span>
<span class="sd">    vision tasks: face attribute recognition, face detection, landmark (or facial part)</span>
<span class="sd">    localization, and face editing &amp; synthesis.</span>

<span class="sd">    Original CelebA dataset structure:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── CelebA</span>
<span class="sd">             ├── README.md</span>
<span class="sd">             ├── Img</span>
<span class="sd">             │    ├── img_celeba.7z</span>
<span class="sd">             │    ├── img_align_celeba_png.7z</span>
<span class="sd">             │    └── img_align_celeba.zip</span>
<span class="sd">             ├── Eval</span>
<span class="sd">             │    └── list_eval_partition.txt</span>
<span class="sd">             └── Anno</span>
<span class="sd">                  ├── list_landmarks_celeba.txt</span>
<span class="sd">                  ├── list_landmarks_align_celeba.txt</span>
<span class="sd">                  ├── list_bbox_celeba.txt</span>
<span class="sd">                  ├── list_attr_celeba.txt</span>
<span class="sd">                  └── identity_CelebA.txt</span>

<span class="sd">    You can unzip the dataset files into the following structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── celeba_dataset_directory</span>
<span class="sd">            ├── list_attr_celeba.txt</span>
<span class="sd">            ├── 000001.jpg</span>
<span class="sd">            ├── 000002.jpg</span>
<span class="sd">            ├── 000003.jpg</span>
<span class="sd">            ├── ...</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @article{DBLP:journals/corr/LiuLWT14,</span>
<span class="sd">        author        = {Ziwei Liu and Ping Luo and Xiaogang Wang and Xiaoou Tang},</span>
<span class="sd">        title         = {Deep Learning Face Attributes in the Wild},</span>
<span class="sd">        journal       = {CoRR},</span>
<span class="sd">        volume        = {abs/1411.7766},</span>
<span class="sd">        year          = {2014},</span>
<span class="sd">        url           = {http://arxiv.org/abs/1411.7766},</span>
<span class="sd">        archivePrefix = {arXiv},</span>
<span class="sd">        eprint        = {1411.7766},</span>
<span class="sd">        timestamp     = {Tue, 10 Dec 2019 15:37:26 +0100},</span>
<span class="sd">        biburl        = {https://dblp.org/rec/journals/corr/LiuLWT14.bib},</span>
<span class="sd">        bibsource     = {dblp computer science bibliography, https://dblp.org},</span>
<span class="sd">        howpublished  = {http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_celebadataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="s1">&#39;all&#39;</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
                 <span class="n">extensions</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">extensions</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">extensions</span><span class="p">,</span> <span class="p">[])</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">!=</span> <span class="s2">&quot;all&quot;</span><span class="p">:</span>
            <span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">realpath</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">)</span>
            <span class="n">partition_file</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="s2">&quot;list_eval_partition.txt&quot;</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">partition_file</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">False</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;Partition file can not be found when usage is not &#39;all&#39;.&quot;</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">CelebANode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">extensions</span><span class="p">)</span>



<span class="k">class</span> <span class="nc">Cifar10Dataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses Cifar10 dataset.</span>
<span class="sd">    This api only supports parsing Cifar10 file in binary version now.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is a scalar of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str, optional): Usage of this dataset, can be `train`, `test` or `all` . `train` will read from 50,000</span>
<span class="sd">            train samples, `test` will read from 10,000 test samples, `all` will read from all 60,000 samples</span>
<span class="sd">            (default=None, all samples).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, number set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected</span>
<span class="sd">            order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; cifar10_dataset_dir = &quot;/path/to/cifar10_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Get all samples from CIFAR10 dataset in sequence</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.Cifar10Dataset(dataset_dir=cifar10_dataset_dir, shuffle=False)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Randomly select 350 samples from CIFAR10 dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.Cifar10Dataset(dataset_dir=cifar10_dataset_dir, num_samples=350, shuffle=True)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 3) Get samples from CIFAR10 dataset for shard 0 in a 2-way distributed training</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.Cifar10Dataset(dataset_dir=cifar10_dataset_dir, num_shards=2, shard_id=0)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # In CIFAR10 dataset, each dictionary has keys &quot;image&quot; and &quot;label&quot;</span>

<span class="sd">    About CIFAR-10 dataset:</span>

<span class="sd">    The CIFAR-10 dataset consists of 60000 32x32 colour images in 10 classes,</span>
<span class="sd">    with 6000 images per class. There are 50000 training images and 10000 test images.</span>
<span class="sd">    The 10 different classes represent airplanes, cars, birds, cats, deer, dogs, frogs, horses, ships, and trucks.</span>

<span class="sd">    Here is the original CIFAR-10 dataset structure.</span>
<span class="sd">    You can unzip the dataset files into the following directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── cifar-10-batches-bin</span>
<span class="sd">             ├── data_batch_1.bin</span>
<span class="sd">             ├── data_batch_2.bin</span>
<span class="sd">             ├── data_batch_3.bin</span>
<span class="sd">             ├── data_batch_4.bin</span>
<span class="sd">             ├── data_batch_5.bin</span>
<span class="sd">             ├── test_batch.bin</span>
<span class="sd">             ├── readme.html</span>
<span class="sd">             └── batches.meta.txt</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @techreport{Krizhevsky09,</span>
<span class="sd">        author       = {Alex Krizhevsky},</span>
<span class="sd">        title        = {Learning multiple layers of features from tiny images},</span>
<span class="sd">        institution  = {},</span>
<span class="sd">        year         = {2009},</span>
<span class="sd">        howpublished = {http://www.cs.toronto.edu/~kriz/cifar.html}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_mnist_cifar_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">Cifar10Node</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">Cifar100Dataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses Cifar100 dataset.</span>

<span class="sd">    The generated dataset has three columns :py:obj:`[image, coarse_label, fine_label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`coarse_label` and :py:obj:`fine_labels` are each a scalar of uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str, optional): Usage of this dataset, can be `train`, `test` or `all` . `train` will read from 50,000</span>
<span class="sd">            train samples, `test` will read from 10,000 test samples, `all` will read from all 60,000 samples</span>
<span class="sd">            (default=None, all samples).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, number set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected</span>
<span class="sd">            order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, &#39;num_samples&#39; reflects</span>
<span class="sd">            the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and shuffle</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; cifar100_dataset_dir = &quot;/path/to/cifar100_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Get all samples from CIFAR100 dataset in sequence</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.Cifar100Dataset(dataset_dir=cifar100_dataset_dir, shuffle=False)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Randomly select 350 samples from CIFAR100 dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.Cifar100Dataset(dataset_dir=cifar100_dataset_dir, num_samples=350, shuffle=True)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # In CIFAR100 dataset, each dictionary has 3 keys: &quot;image&quot;, &quot;fine_label&quot; and &quot;coarse_label&quot;</span>

<span class="sd">    About CIFAR-100 dataset:</span>

<span class="sd">    This dataset is just like the CIFAR-10, except it has 100 classes containing 600 images</span>
<span class="sd">    each. There are 500 training images and 100 testing images per class. The 100 classes in</span>
<span class="sd">    the CIFAR-100 are grouped into 20 superclasses. Each image comes with a &quot;fine&quot; label (the</span>
<span class="sd">    class to which it belongs) and a &quot;coarse&quot; label (the superclass to which it belongs).</span>

<span class="sd">    Here is the original CIFAR-100 dataset structure.</span>
<span class="sd">    You can unzip the dataset files into the following directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── cifar-100-binary</span>
<span class="sd">            ├── train.bin</span>
<span class="sd">            ├── test.bin</span>
<span class="sd">            ├── fine_label_names.txt</span>
<span class="sd">            └── coarse_label_names.txt</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @techreport{Krizhevsky09,</span>
<span class="sd">        author       = {Alex Krizhevsky},</span>
<span class="sd">        title        = {Learning multiple layers of features from tiny images},</span>
<span class="sd">        institution  = {},</span>
<span class="sd">        year         = {2009},</span>
<span class="sd">        howpublished = {http://www.cs.toronto.edu/~kriz/cifar.html}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_mnist_cifar_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">Cifar100Node</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">CityscapesDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses Cityscapes dataset.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, task]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`task` is of the uint8 type if task is not &#39;polygon&#39; otherwise task is</span>
<span class="sd">    a string tensor with serialize json.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str): Acceptable usages include `train`, `test`, `val` or `all` if quality_mode is `fine`</span>
<span class="sd">            otherwise `train`, `train_extra`, `val` or `all` (default= `train`).</span>
<span class="sd">        quality_mode (str): Acceptable quality_modes include `fine` or `coarse` (default= `fine`).</span>
<span class="sd">        task (str): Acceptable tasks include `instance`, `semantic`, `polygon` or `color` (default= `instance`).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset.</span>
<span class="sd">            (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, number set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected</span>
<span class="sd">            order behavior shown in the table).</span>
<span class="sd">        decode (bool, optional): Decode the images after reading (default=False).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the max sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir is invalid or does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If dataset_dir is not exist.</span>
<span class="sd">        ValueError: If task is invalid.</span>
<span class="sd">        ValueError: If quality_mode is invalid.</span>
<span class="sd">        ValueError: If usage is invalid.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; cityscapes_dataset_dir = &quot;/path/to/cityscapes_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Get all samples from Cityscapes dataset in sequence</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.CityscapesDataset(dataset_dir=cityscapes_dataset_dir, task=&quot;instance&quot;, quality_mode=&quot;fine&quot;,</span>
<span class="sd">        ...                                usage=&quot;train&quot;, shuffle=False, num_parallel_workers=1)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Randomly select 350 samples from Cityscapes dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.CityscapesDataset(dataset_dir=cityscapes_dataset_dir, num_samples=350, shuffle=True,</span>
<span class="sd">        ...                                num_parallel_workers=1)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 3) Get samples from Cityscapes dataset for shard 0 in a 2-way distributed training</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.CityscapesDataset(dataset_dir=cityscapes_dataset_dir, num_shards=2, shard_id=0,</span>
<span class="sd">        ...                                num_parallel_workers=1)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # In Cityscapes dataset, each dictionary has keys &quot;image&quot; and &quot;task&quot;</span>

<span class="sd">    About Cityscapes dataset:</span>

<span class="sd">    The Cityscapes dataset consists of 5000 colour images with high quality dense pixel annotations and</span>
<span class="sd">    19998 colour images with coarser polygonal annotations in 50 cities. There are 30 classes in this</span>
<span class="sd">    dataset and the polygonal annotations include dense semantic segmentation and instance segmentation</span>
<span class="sd">    for vehicle and people.</span>

<span class="sd">    You can unzip the dataset files into the following directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    Taking the quality_mode of `fine` as an example.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── Cityscapes</span>
<span class="sd">             ├── leftImg8bit</span>
<span class="sd">             |    ├── train</span>
<span class="sd">             |    |    ├── aachen</span>
<span class="sd">             |    |    |    ├── aachen_000000_000019_leftImg8bit.png</span>
<span class="sd">             |    |    |    ├── aachen_000001_000019_leftImg8bit.png</span>
<span class="sd">             |    |    |    ├── ...</span>
<span class="sd">             |    |    ├── bochum</span>
<span class="sd">             |    |    |    ├── ...</span>
<span class="sd">             |    |    ├── ...</span>
<span class="sd">             |    ├── test</span>
<span class="sd">             |    |    ├── ...</span>
<span class="sd">             |    ├── val</span>
<span class="sd">             |    |    ├── ...</span>
<span class="sd">             └── gtFine</span>
<span class="sd">                  ├── train</span>
<span class="sd">                  |    ├── aachen</span>
<span class="sd">                  |    |    ├── aachen_000000_000019_gtFine_color.png</span>
<span class="sd">                  |    |    ├── aachen_000000_000019_gtFine_instanceIds.png</span>
<span class="sd">                  |    |    ├── aachen_000000_000019_gtFine_labelIds.png</span>
<span class="sd">                  |    |    ├── aachen_000000_000019_gtFine_polygons.json</span>
<span class="sd">                  |    |    ├── aachen_000001_000019_gtFine_color.png</span>
<span class="sd">                  |    |    ├── aachen_000001_000019_gtFine_instanceIds.png</span>
<span class="sd">                  |    |    ├── aachen_000001_000019_gtFine_labelIds.png</span>
<span class="sd">                  |    |    ├── aachen_000001_000019_gtFine_polygons.json</span>
<span class="sd">                  |    |    ├── ...</span>
<span class="sd">                  |    ├── bochum</span>
<span class="sd">                  |    |    ├── ...</span>
<span class="sd">                  |    ├── ...</span>
<span class="sd">                  ├── test</span>
<span class="sd">                  |    ├── ...</span>
<span class="sd">                  └── val</span>
<span class="sd">                       ├── ...</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @inproceedings{Cordts2016Cityscapes,</span>
<span class="sd">        title       = {The Cityscapes Dataset for Semantic Urban Scene Understanding},</span>
<span class="sd">        author      = {Cordts, Marius and Omran, Mohamed and Ramos, Sebastian and Rehfeld, Timo and Enzweiler,</span>
<span class="sd">                        Markus and Benenson, Rodrigo and Franke, Uwe and Roth, Stefan and Schiele, Bernt},</span>
<span class="sd">        booktitle   = {Proc. of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},</span>
<span class="sd">        year        = {2016}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_cityscapes_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="s2">&quot;train&quot;</span><span class="p">,</span> <span class="n">quality_mode</span><span class="o">=</span><span class="s2">&quot;fine&quot;</span><span class="p">,</span> <span class="n">task</span><span class="o">=</span><span class="s2">&quot;instance&quot;</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">task</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">quality_mode</span> <span class="o">=</span> <span class="n">quality_mode</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">usage</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">CityscapesNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">quality_mode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">CocoDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses COCO dataset.</span>

<span class="sd">    CocoDataset supports four kinds of tasks, which are Object Detection, Keypoint Detection, Stuff Segmentation and</span>
<span class="sd">    Panoptic Segmentation of 2017 Train/Val/Test dataset.</span>

<span class="sd">    The generated dataset with different task setting has different output columns:</span>

<span class="sd">    - task = :py:obj:`Detection`, output columns: :py:obj:`[image, dtype=uint8]`, :py:obj:`[bbox, dtype=float32]`, \</span>
<span class="sd">        :py:obj:`[category_id, dtype=uint32]`, :py:obj:`[iscrowd, dtype=uint32]`.</span>
<span class="sd">    - task = :py:obj:`Stuff`, output columns: :py:obj:`[image, dtype=uint8]`, :py:obj:`[segmentation,dtype=float32]`, \</span>
<span class="sd">        :py:obj:`[iscrowd,dtype=uint32]`.</span>
<span class="sd">    - task = :py:obj:`Keypoint`, output columns: :py:obj:`[image, dtype=uint8]`, \</span>
<span class="sd">        :py:obj:`[keypoints, dtype=float32]`, :py:obj:`[num_keypoints, dtype=uint32]`.</span>
<span class="sd">    - task = :py:obj:`Panoptic`, output columns: :py:obj:`[image, dtype=uint8]`, :py:obj:`[bbox, dtype=float32]`, \</span>
<span class="sd">        :py:obj:`[category_id, dtype=uint32]`, :py:obj:`[iscrowd, dtype=uint32]`, :py:obj:`[area, dtype=uint32]`.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        annotation_file (str): Path to the annotation JSON file.</span>
<span class="sd">        task (str, optional): Set the task type for reading COCO data. Supported task types:</span>
<span class="sd">            `Detection`, `Stuff`, `Panoptic` and `Keypoint` (default= `Detection`).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, number set in the configuration file).</span>
<span class="sd">        shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected</span>
<span class="sd">            order behavior shown in the table).</span>
<span class="sd">        decode (bool, optional): Decode the images after reading (default=False).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>
<span class="sd">        extra_metadata(bool, optional): Flag to add extra meta-data to row. If True, an additional column will be</span>
<span class="sd">            output at the end :py:obj:`[_meta-filename, dtype=string]` (default=False).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        RuntimeError: If parse JSON file failed.</span>
<span class="sd">        ValueError: If task is not in [`Detection`, `Stuff`, `Panoptic`, `Keypoint`].</span>
<span class="sd">        ValueError: If annotation_file is not exist.</span>
<span class="sd">        ValueError: If dataset_dir is not exist.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - Column &#39;[_meta-filename, dtype=string]&#39; won&#39;t be output unless an explicit rename dataset op is added</span>
<span class="sd">          to remove the prefix(&#39;_meta-&#39;).</span>
<span class="sd">        - CocoDataset doesn&#39;t support PKSampler.</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; coco_dataset_dir = &quot;/path/to/coco_dataset_directory/images&quot;</span>
<span class="sd">        &gt;&gt;&gt; coco_annotation_file = &quot;/path/to/coco_dataset_directory/annotation_file&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Read COCO data for Detection task</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.CocoDataset(dataset_dir=coco_dataset_dir,</span>
<span class="sd">        ...                          annotation_file=coco_annotation_file,</span>
<span class="sd">        ...                          task=&#39;Detection&#39;)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Read COCO data for Stuff task</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.CocoDataset(dataset_dir=coco_dataset_dir,</span>
<span class="sd">        ...                          annotation_file=coco_annotation_file,</span>
<span class="sd">        ...                          task=&#39;Stuff&#39;)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 3) Read COCO data for Panoptic task</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.CocoDataset(dataset_dir=coco_dataset_dir,</span>
<span class="sd">        ...                          annotation_file=coco_annotation_file,</span>
<span class="sd">        ...                          task=&#39;Panoptic&#39;)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 4) Read COCO data for Keypoint task</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.CocoDataset(dataset_dir=coco_dataset_dir,</span>
<span class="sd">        ...                          annotation_file=coco_annotation_file,</span>
<span class="sd">        ...                          task=&#39;Keypoint&#39;)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # In COCO dataset, each dictionary has keys &quot;image&quot; and &quot;annotation&quot;</span>

<span class="sd">    About COCO dataset:</span>

<span class="sd">    COCO(Microsoft Common Objects in Context) is a large-scale object detection, segmentation, and captioning dataset</span>
<span class="sd">    with several features: Object segmentation, Recognition in context, Superpixel stuff segmentation,</span>
<span class="sd">    330K images (&gt;200K labeled), 1.5 million object instances, 80 object categories, 91 stuff categories,</span>
<span class="sd">    5 captions per image, 250,000 people with keypoints. In contrast to the popular ImageNet dataset, COCO has fewer</span>
<span class="sd">    categories but more instances in per category.</span>

<span class="sd">    You can unzip the original COCO-2017 dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── coco_dataset_directory</span>
<span class="sd">             ├── train2017</span>
<span class="sd">             │    ├── 000000000009.jpg</span>
<span class="sd">             │    ├── 000000000025.jpg</span>
<span class="sd">             │    ├── ...</span>
<span class="sd">             ├── test2017</span>
<span class="sd">             │    ├── 000000000001.jpg</span>
<span class="sd">             │    ├── 000000058136.jpg</span>
<span class="sd">             │    ├── ...</span>
<span class="sd">             ├── val2017</span>
<span class="sd">             │    ├── 000000000139.jpg</span>
<span class="sd">             │    ├── 000000057027.jpg</span>
<span class="sd">             │    ├── ...</span>
<span class="sd">             └── annotations</span>
<span class="sd">                  ├── captions_train2017.json</span>
<span class="sd">                  ├── captions_val2017.json</span>
<span class="sd">                  ├── instances_train2017.json</span>
<span class="sd">                  ├── instances_val2017.json</span>
<span class="sd">                  ├── person_keypoints_train2017.json</span>
<span class="sd">                  └── person_keypoints_val2017.json</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @article{DBLP:journals/corr/LinMBHPRDZ14,</span>
<span class="sd">        author        = {Tsung{-}Yi Lin and Michael Maire and Serge J. Belongie and</span>
<span class="sd">                        Lubomir D. Bourdev and  Ross B. Girshick and James Hays and</span>
<span class="sd">                        Pietro Perona and Deva Ramanan and Piotr Doll{\&#39;{a}}r and C. Lawrence Zitnick},</span>
<span class="sd">        title         = {Microsoft {COCO:} Common Objects in Context},</span>
<span class="sd">        journal       = {CoRR},</span>
<span class="sd">        volume        = {abs/1405.0312},</span>
<span class="sd">        year          = {2014},</span>
<span class="sd">        url           = {http://arxiv.org/abs/1405.0312},</span>
<span class="sd">        archivePrefix = {arXiv},</span>
<span class="sd">        eprint        = {1405.0312},</span>
<span class="sd">        timestamp     = {Mon, 13 Aug 2018 16:48:13 +0200},</span>
<span class="sd">        biburl        = {https://dblp.org/rec/journals/corr/LinMBHPRDZ14.bib},</span>
<span class="sd">        bibsource     = {dblp computer science bibliography, https://dblp.org}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_cocodataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">annotation_file</span><span class="p">,</span> <span class="n">task</span><span class="o">=</span><span class="s2">&quot;Detection&quot;</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">extra_metadata</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">annotation_file</span> <span class="o">=</span> <span class="n">annotation_file</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="s2">&quot;Detection&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">extra_metadata</span> <span class="o">=</span> <span class="n">extra_metadata</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">CocoNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">annotation_file</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">,</span>
                            <span class="bp">self</span><span class="o">.</span><span class="n">extra_metadata</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">get_class_indexing</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Get the class index.</span>

<span class="sd">        Returns:</span>
<span class="sd">            dict, a str-to-list&lt;int&gt; mapping from label name to index.</span>

<span class="sd">        Examples:</span>
<span class="sd">            &gt;&gt;&gt; coco_dataset_dir = &quot;/path/to/coco_dataset_directory/images&quot;</span>
<span class="sd">            &gt;&gt;&gt; coco_annotation_file = &quot;/path/to/coco_dataset_directory/annotation_file&quot;</span>
<span class="sd">            &gt;&gt;&gt;</span>
<span class="sd">            &gt;&gt;&gt; # Read COCO data for Detection task</span>
<span class="sd">            &gt;&gt;&gt; dataset = ds.CocoDataset(dataset_dir=coco_dataset_dir,</span>
<span class="sd">            ...                          annotation_file=coco_annotation_file,</span>
<span class="sd">            ...                          task=&#39;Detection&#39;)</span>
<span class="sd">            &gt;&gt;&gt;</span>
<span class="sd">            &gt;&gt;&gt; class_indexing = dataset.get_class_indexing()</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">{</span><span class="s2">&quot;Detection&quot;</span><span class="p">,</span> <span class="s2">&quot;Panoptic&quot;</span><span class="p">}:</span>
            <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">&quot;Only &#39;Detection&#39; and &#39;Panoptic&#39; support get_class_indexing.&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_class_indexing</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">runtime_getter</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_init_tree_getters</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_class_indexing</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">runtime_getter</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">GetClassIndexing</span><span class="p">())</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_class_indexing</span>


<span class="k">class</span> <span class="nc">DIV2KDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses DIV2KDataset dataset.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[hr_image, lr_image]`.</span>
<span class="sd">    The tensor of column :py:obj:`hr_image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`lr_image` is of the uint8 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str): Acceptable usages include `train`, `valid` or `all` (default= `train`).</span>
<span class="sd">        downgrade (str): Acceptable downgrades include `bicubic`, `unknown`, `mild`, `difficult` or</span>
<span class="sd">            `wild` (default= `bicubic`).</span>
<span class="sd">        scale (int): Acceptable scales include 2, 3, 4 or 8 (default=2).</span>
<span class="sd">            When `downgrade` is `bicubic`, scale can be 2, 3, 4, 8.</span>
<span class="sd">            When `downgrade` is `unknown`, scale can only be 2, 3, 4.</span>
<span class="sd">            When `downgrade` is `mild`, `difficult` or `wild`, scale can only be 4.</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset.</span>
<span class="sd">            (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, number set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected</span>
<span class="sd">            order behavior shown in the table).</span>
<span class="sd">        decode (bool, optional): Decode the images after reading (default=False).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the max sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir is invalid or does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If dataset_dir is not exist.</span>
<span class="sd">        ValueError: If usage is invalid.</span>
<span class="sd">        ValueError: If downgrade is invalid.</span>
<span class="sd">        ValueError: If scale is invalid.</span>
<span class="sd">        ValueError: If scale equal to 8 and downgrade not equal to `bicubic`.</span>
<span class="sd">        ValueError: If downgrade in [`mild`, `difficult`, `wild`] and scale not equal to 4.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; div2k_dataset_dir = &quot;/path/to/div2k_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Get all samples from DIV2K dataset in sequence</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.DIV2KDataset(dataset_dir=div2k_dataset_dir, usage=&quot;train&quot;, scale=2, downgrade=&quot;bicubic&quot;,</span>
<span class="sd">        ...                           shuffle=False)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Randomly select 350 samples from DIV2K dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.DIV2KDataset(dataset_dir=div2k_dataset_dir, usage=&quot;train&quot;, scale=2, downgrade=&quot;bicubic&quot;,</span>
<span class="sd">        ...                           num_samples=350, shuffle=True)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 3) Get samples from DIV2K dataset for shard 0 in a 2-way distributed training</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.DIV2KDataset(dataset_dir=div2k_dataset_dir, usage=&quot;train&quot;, scale=2, downgrade=&quot;bicubic&quot;,</span>
<span class="sd">        ...                           num_shards=2, shard_id=0)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # In DIV2K dataset, each dictionary has keys &quot;hr_image&quot; and &quot;lr_image&quot;</span>

<span class="sd">    About DIV2K dataset:</span>

<span class="sd">    The DIV2K dataset consists of 1000 2K resolution images, among which 800 images are for training, 100 images</span>
<span class="sd">    are for validation and 100 images are for testing. NTIRE 2017 and NTIRE 2018 include only training dataset</span>
<span class="sd">    and validation dataset.</span>

<span class="sd">    You can unzip the dataset files into the following directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    Take the training set as an example.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── DIV2K</span>
<span class="sd">             ├── DIV2K_train_HR</span>
<span class="sd">             |    ├── 0001.png</span>
<span class="sd">             |    ├── 0002.png</span>
<span class="sd">             |    ├── ...</span>
<span class="sd">             ├── DIV2K_train_LR_bicubic</span>
<span class="sd">             |    ├── X2</span>
<span class="sd">             |    |    ├── 0001x2.png</span>
<span class="sd">             |    |    ├── 0002x2.png</span>
<span class="sd">             |    |    ├── ...</span>
<span class="sd">             |    ├── X3</span>
<span class="sd">             |    |    ├── 0001x3.png</span>
<span class="sd">             |    |    ├── 0002x3.png</span>
<span class="sd">             |    |    ├── ...</span>
<span class="sd">             |    └── X4</span>
<span class="sd">             |         ├── 0001x4.png</span>
<span class="sd">             |         ├── 0002x4.png</span>
<span class="sd">             |         ├── ...</span>
<span class="sd">             ├── DIV2K_train_LR_unknown</span>
<span class="sd">             |    ├── X2</span>
<span class="sd">             |    |    ├── 0001x2.png</span>
<span class="sd">             |    |    ├── 0002x2.png</span>
<span class="sd">             |    |    ├── ...</span>
<span class="sd">             |    ├── X3</span>
<span class="sd">             |    |    ├── 0001x3.png</span>
<span class="sd">             |    |    ├── 0002x3.png</span>
<span class="sd">             |    |    ├── ...</span>
<span class="sd">             |    └── X4</span>
<span class="sd">             |         ├── 0001x4.png</span>
<span class="sd">             |         ├── 0002x4.png</span>
<span class="sd">             |         ├── ...</span>
<span class="sd">             ├── DIV2K_train_LR_mild</span>
<span class="sd">             |    ├── 0001x4m.png</span>
<span class="sd">             |    ├── 0002x4m.png</span>
<span class="sd">             |    ├── ...</span>
<span class="sd">             ├── DIV2K_train_LR_difficult</span>
<span class="sd">             |    ├── 0001x4d.png</span>
<span class="sd">             |    ├── 0002x4d.png</span>
<span class="sd">             |    ├── ...</span>
<span class="sd">             ├── DIV2K_train_LR_wild</span>
<span class="sd">             |    ├── 0001x4w.png</span>
<span class="sd">             |    ├── 0002x4w.png</span>
<span class="sd">             |    ├── ...</span>
<span class="sd">             └── DIV2K_train_LR_x8</span>
<span class="sd">                  ├── 0001x8.png</span>
<span class="sd">                  ├── 0002x8.png</span>
<span class="sd">                  ├── ...</span>
<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @InProceedings{Agustsson_2017_CVPR_Workshops,</span>
<span class="sd">        author    = {Agustsson, Eirikur and Timofte, Radu},</span>
<span class="sd">        title     = {NTIRE 2017 Challenge on Single Image Super-Resolution: Dataset and Study},</span>
<span class="sd">        booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR) Workshops},</span>
<span class="sd">        url       = &quot;http://www.vision.ee.ethz.ch/~timofter/publications/Agustsson-CVPRW-2017.pdf&quot;,</span>
<span class="sd">        month     = {July},</span>
<span class="sd">        year      = {2017}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_div2k_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="s2">&quot;train&quot;</span><span class="p">,</span> <span class="n">downgrade</span><span class="o">=</span><span class="s2">&quot;bicubic&quot;</span><span class="p">,</span> <span class="n">scale</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">usage</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="o">=</span> <span class="n">scale</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">downgrade</span> <span class="o">=</span> <span class="n">downgrade</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">DIV2KNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">downgrade</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span>


<div class="viewcode-block" id="EMnistDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.EMnistDataset.html#mindspore.dataset.EMnistDataset">[docs]</a><span class="k">class</span> <span class="nc">EMnistDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses the EMNIST dataset.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is a scalar of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        name (str): Name of splits for this dataset, can be &quot;byclass&quot;, &quot;bymerge&quot;, &quot;balanced&quot;, &quot;letters&quot;, &quot;digits&quot;</span>
<span class="sd">            or &quot;mnist&quot;.</span>
<span class="sd">        usage (str, optional): Usage of this dataset, can be &quot;train&quot;, &quot;test&quot; or &quot;all&quot;.</span>
<span class="sd">            (default=None, will read all samples).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, will read all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, will use value set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            When this argument is specified, `num_samples` reflects the max sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This</span>
<span class="sd">            argument can only be specified when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; emnist_dataset_dir = &quot;/path/to/emnist_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Read 3 samples from EMNIST dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.EMnistDataset(dataset_dir=emnist_dataset_dir, name=&quot;mnist&quot;, num_samples=3)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Note: In emnist_dataset dataset, each dictionary has keys &quot;image&quot; and &quot;label&quot;</span>

<span class="sd">    About EMNIST dataset:</span>

<span class="sd">    The EMNIST dataset is a set of handwritten character digits derived from the NIST Special</span>
<span class="sd">    Database 19 and converted to a 28x28 pixel image format and dataset structure that directly</span>
<span class="sd">    matches the MNIST dataset. Further information on the dataset contents and conversion process</span>
<span class="sd">    can be found in the paper available at https://arxiv.org/abs/1702.05373v1.</span>

<span class="sd">    The numbers of characters and classes of each split of EMNIST are as follows:</span>

<span class="sd">    By Class: 814,255 characters and 62 unbalanced classes.</span>
<span class="sd">    By Merge: 814,255 characters and 47 unbalanced classes.</span>
<span class="sd">    Balanced: 131,600 characters and 47 balanced classes.</span>
<span class="sd">    Letters: 145,600 characters and 26 balanced classes.</span>
<span class="sd">    Digits: 280,000 characters and 10 balanced classes.</span>
<span class="sd">    MNIST: 70,000 characters and 10 balanced classes.</span>

<span class="sd">    Here is the original EMNIST dataset structure.</span>
<span class="sd">    You can unzip the dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── mnist_dataset_dir</span>
<span class="sd">             ├── emnist-mnist-train-images-idx3-ubyte</span>
<span class="sd">             ├── emnist-mnist-train-labels-idx1-ubyte</span>
<span class="sd">             ├── emnist-mnist-test-images-idx3-ubyte</span>
<span class="sd">             ├── emnist-mnist-test-labels-idx1-ubyte</span>
<span class="sd">             ├── ...</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @article{cohen_afshar_tapson_schaik_2017,</span>
<span class="sd">        title        = {EMNIST: Extending MNIST to handwritten letters},</span>
<span class="sd">        DOI          = {10.1109/ijcnn.2017.7966217},</span>
<span class="sd">        journal      = {2017 International Joint Conference on Neural Networks (IJCNN)},</span>
<span class="sd">        author       = {Cohen, Gregory and Afshar, Saeed and Tapson, Jonathan and Schaik, Andre Van},</span>
<span class="sd">        year         = {2017},</span>
<span class="sd">        howpublished = {https://www.westernsydney.edu.au/icns/reproducible_research/</span>
<span class="sd">                        publication_support_materials/emnist}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_emnist_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">name</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">EMnistNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span></div>


<div class="viewcode-block" id="FakeImageDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.FakeImageDataset.html#mindspore.dataset.FakeImageDataset">[docs]</a><span class="k">class</span> <span class="nc">FakeImageDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset for generating fake images.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is a scalar of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        num_images (int, optional): Number of images to generate in the dataset (default=1000).</span>
<span class="sd">        image_size (tuple, optional):  Size of the fake image (default=(224, 224, 3)).</span>
<span class="sd">        num_classes (int, optional): Number of classes in the dataset (default=10).</span>
<span class="sd">        base_seed (int, optional): Offsets the index-based random seed used to generate each image (default=0).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, will read all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, will use value set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            When this argument is specified, `num_samples` reflects the max sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This</span>
<span class="sd">            argument can only be specified when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a sampler. &#39;sampler&#39; and &#39;shuffle&#39; are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using &#39;sampler&#39; and &#39;shuffle&#39;</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter &#39;sampler&#39;</span>
<span class="sd">         - Parameter &#39;shuffle&#39;</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; # Read 3 samples from FakeImage dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.FakeImageDataset(num_images=1000, image_size=(224,224,3),</span>
<span class="sd">        ...                               num_classes=10, base_seed=0, num_samples=3)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Note: In FakeImage dataset, each dictionary has keys &quot;image&quot; and &quot;label&quot;</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_fake_image_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num_images</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span> <span class="n">image_size</span><span class="o">=</span><span class="p">(</span><span class="mi">224</span><span class="p">,</span> <span class="mi">224</span><span class="p">,</span> <span class="mi">3</span><span class="p">),</span> <span class="n">num_classes</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">base_seed</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">num_images</span> <span class="o">=</span> <span class="n">num_images</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">image_size</span> <span class="o">=</span> <span class="n">image_size</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_classes</span> <span class="o">=</span> <span class="n">num_classes</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">base_seed</span> <span class="o">=</span> <span class="n">base_seed</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">FakeImageNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">num_images</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_classes</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_seed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span></div>


<div class="viewcode-block" id="FashionMnistDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.FashionMnistDataset.html#mindspore.dataset.FashionMnistDataset">[docs]</a><span class="k">class</span> <span class="nc">FashionMnistDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses the FASHION-MNIST dataset.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is a scalar of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str, optional): Usage of this dataset, can be `train`, `test` or `all`. `train` will read from 60,000</span>
<span class="sd">            train samples, `test` will read from 10,000 test samples, `all` will read from all 70,000 samples.</span>
<span class="sd">            (default=None, will read all samples)</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, will read all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, will use value set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            When this argument is specified, `num_samples` reflects the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This</span>
<span class="sd">            argument can only be specified when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; fashion_mnist_dataset_dir = &quot;/path/to/fashion_mnist_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Read 3 samples from FASHIONMNIST dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.FashionMnistDataset(dataset_dir=fashion_mnist_dataset_dir, num_samples=3)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Note: In FASHIONMNIST dataset, each dictionary has keys &quot;image&quot; and &quot;label&quot;</span>

<span class="sd">    About Fashion-MNIST dataset:</span>

<span class="sd">    Fashion-MNIST is a dataset of Zalando&#39;s article images—consisting of a training set of 60,000 examples and</span>
<span class="sd">    a test set of 10,000 examples. Each example is a 28x28 grayscale image, associated with a label from 10 classes.</span>
<span class="sd">    We intend Fashion-MNIST to serve as a direct drop-in replacement for the original MNIST dataset for benchmarking</span>
<span class="sd">    machine learning algorithms. It shares the same image size and structure of training and testing splits.</span>

<span class="sd">    You can unzip the dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── fashionmnist_dataset_dir</span>
<span class="sd">             ├── t10k-images-idx3-ubyte</span>
<span class="sd">             ├── t10k-labels-idx1-ubyte</span>
<span class="sd">             ├── train-images-idx3-ubyte</span>
<span class="sd">             └── train-labels-idx1-ubyte</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @online{xiao2017/online,</span>
<span class="sd">          author       = {Han Xiao and Kashif Rasul and Roland Vollgraf},</span>
<span class="sd">          title        = {Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms},</span>
<span class="sd">          date         = {2017-08-28},</span>
<span class="sd">          year         = {2017},</span>
<span class="sd">          eprintclass  = {cs.LG},</span>
<span class="sd">          eprinttype   = {arXiv},</span>
<span class="sd">          eprint       = {cs.LG/1708.07747},</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_mnist_cifar_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">FashionMnistNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span></div>


<div class="viewcode-block" id="FlickrDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.FlickrDataset.html#mindspore.dataset.FlickrDataset">[docs]</a><span class="k">class</span> <span class="nc">FlickrDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses Flickr8k and Flickr30k dataset.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, annotation]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`annotation` is a tensor which contains 5 annotations string,</span>
<span class="sd">    such as [&quot;a&quot;, &quot;b&quot;, &quot;c&quot;, &quot;d&quot;, &quot;e&quot;].</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        annotation_file (str): Path to the root directory that contains the annotation.</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset.</span>
<span class="sd">            (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, number set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected</span>
<span class="sd">            order behavior shown in the table).</span>
<span class="sd">        decode (bool, optional): Decode the images after reading (default=False).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the max sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir is not valid or does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If dataset_dir is not exist.</span>
<span class="sd">        ValueError: If annotation_file is not exist.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; flickr_dataset_dir = &quot;/path/to/flickr_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt; annotation_file = &quot;/path/to/flickr_annotation_file&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Get all samples from FLICKR dataset in sequence</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.FlickrDataset(dataset_dir=flickr_dataset_dir,</span>
<span class="sd">        ...                            annotation_file=annotation_file,</span>
<span class="sd">        ...                            shuffle=False)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Randomly select 350 samples from FLICKR dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.FlickrDataset(dataset_dir=flickr_dataset_dir,</span>
<span class="sd">        ...                            annotation_file=annotation_file,</span>
<span class="sd">        ...                            num_samples=350,</span>
<span class="sd">        ...                            shuffle=True)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 3) Get samples from FLICKR dataset for shard 0 in a 2-way distributed training</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.FlickrDataset(dataset_dir=flickr_dataset_dir,</span>
<span class="sd">        ...                            annotation_file=annotation_file,</span>
<span class="sd">        ...                            num_shards=2,</span>
<span class="sd">        ...                            shard_id=0)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # In FLICKR dataset, each dictionary has keys &quot;image&quot; and &quot;annotation&quot;</span>

<span class="sd">    About Flickr8k dataset:</span>

<span class="sd">    The Flickr8k dataset consists of 8092 colour images. There are 40460 annotations in the Flickr8k.token.txt,</span>
<span class="sd">    each image has 5 annotations.</span>

<span class="sd">    You can unzip the dataset files into the following directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── Flickr8k</span>
<span class="sd">             ├── Flickr8k_Dataset</span>
<span class="sd">             │    ├── 1000268201_693b08cb0e.jpg</span>
<span class="sd">             │    ├── 1001773457_577c3a7d70.jpg</span>
<span class="sd">             │    ├── ...</span>
<span class="sd">             └── Flickr8k.token.txt</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @article{DBLP:journals/jair/HodoshYH13,</span>
<span class="sd">        author    = {Micah Hodosh and Peter Young and Julia Hockenmaier},</span>
<span class="sd">        title     = {Framing Image Description as a Ranking Task: Data, Models and Evaluation Metrics},</span>
<span class="sd">        journal   = {J. Artif. Intell. Res.},</span>
<span class="sd">        volume    = {47},</span>
<span class="sd">        pages     = {853--899},</span>
<span class="sd">        year      = {2013},</span>
<span class="sd">        url       = {https://doi.org/10.1613/jair.3994},</span>
<span class="sd">        doi       = {10.1613/jair.3994},</span>
<span class="sd">        timestamp = {Mon, 21 Jan 2019 15:01:17 +0100},</span>
<span class="sd">        biburl    = {https://dblp.org/rec/journals/jair/HodoshYH13.bib},</span>
<span class="sd">        bibsource = {dblp computer science bibliography, https://dblp.org}</span>
<span class="sd">        }</span>

<span class="sd">    About Flickr30k dataset:</span>

<span class="sd">    The Flickr30k dataset consists of 31783 colour images. There are 158915 annotations in</span>
<span class="sd">    the results_20130124.token, each image has 5 annotations.</span>

<span class="sd">    You can unzip the dataset files into the following directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── Flickr30k</span>
<span class="sd">             ├── flickr30k-images</span>
<span class="sd">             │    ├── 1000092795.jpg</span>
<span class="sd">             │    ├── 10002456.jpg</span>
<span class="sd">             │    ├── ...</span>
<span class="sd">             └── results_20130124.token</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @article{DBLP:journals/tacl/YoungLHH14,</span>
<span class="sd">        author    = {Peter Young and Alice Lai and Micah Hodosh and Julia Hockenmaier},</span>
<span class="sd">        title     = {From image descriptions to visual denotations: New similarity metrics</span>
<span class="sd">                     for semantic inference over event descriptions},</span>
<span class="sd">        journal   = {Trans. Assoc. Comput. Linguistics},</span>
<span class="sd">        volume    = {2},</span>
<span class="sd">        pages     = {67--78},</span>
<span class="sd">        year      = {2014},</span>
<span class="sd">        url       = {https://tacl2013.cs.columbia.edu/ojs/index.php/tacl/article/view/229},</span>
<span class="sd">        timestamp = {Wed, 17 Feb 2021 21:55:25 +0100},</span>
<span class="sd">        biburl    = {https://dblp.org/rec/journals/tacl/YoungLHH14.bib},</span>
<span class="sd">        bibsource = {dblp computer science bibliography, https://dblp.org}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_flickr_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">annotation_file</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">decode</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">annotation_file</span> <span class="o">=</span> <span class="n">annotation_file</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">FlickrNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">annotation_file</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span></div>


<span class="k">class</span> <span class="nc">_Flowers102Dataset</span><span class="p">:</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Mainly for loading Flowers102 Dataset, and return one row each time.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="n">usage</span><span class="p">,</span> <span class="n">decode</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">realpath</span><span class="p">(</span><span class="n">dataset_dir</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">task</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">usage</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">decode</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">==</span> <span class="s2">&quot;Classification&quot;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">column_names</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">,</span> <span class="s2">&quot;label&quot;</span><span class="p">]</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">column_names</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">,</span> <span class="s2">&quot;segmentation&quot;</span><span class="p">,</span> <span class="s2">&quot;label&quot;</span><span class="p">]</span>

        <span class="n">labels_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="s2">&quot;imagelabels.mat&quot;</span><span class="p">)</span>
        <span class="n">setid_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="s2">&quot;setid.mat&quot;</span><span class="p">)</span>
        <span class="c1"># minus one to transform 1~102 to 0 ~ 101</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="p">(</span><span class="n">loadmat</span><span class="p">(</span><span class="n">labels_path</span><span class="p">)[</span><span class="s2">&quot;labels&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">uint32</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">setid</span> <span class="o">=</span> <span class="n">loadmat</span><span class="p">(</span><span class="n">setid_path</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">==</span> <span class="s1">&#39;train&#39;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">indices</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">setid</span><span class="p">[</span><span class="s2">&quot;trnid&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">==</span> <span class="s1">&#39;test&#39;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">indices</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">setid</span><span class="p">[</span><span class="s2">&quot;tstid&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">==</span> <span class="s1">&#39;valid&#39;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">indices</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">setid</span><span class="p">[</span><span class="s2">&quot;valid&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">==</span> <span class="s1">&#39;all&#39;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">indices</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">setid</span><span class="p">[</span><span class="s2">&quot;trnid&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">indices</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">setid</span><span class="p">[</span><span class="s2">&quot;tstid&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">indices</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">setid</span><span class="p">[</span><span class="s2">&quot;valid&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Input usage is not within the valid set of [&#39;train&#39;, &#39;valid&#39;, &#39;test&#39;, &#39;all&#39;].&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="fm">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">index</span><span class="p">):</span>
        <span class="c1"># range: 1 ~ 8189</span>
        <span class="n">image_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="s2">&quot;jpg&quot;</span><span class="p">,</span> <span class="s2">&quot;image_&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">[</span><span class="n">index</span><span class="p">])</span><span class="o">.</span><span class="n">zfill</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;.jpg&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">image_path</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;Can not find image file: &quot;</span> <span class="o">+</span> <span class="n">image_path</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
            <span class="n">image</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">Image</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">image_path</span><span class="p">)</span><span class="o">.</span><span class="n">convert</span><span class="p">(</span><span class="s2">&quot;RGB&quot;</span><span class="p">))</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">image</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">fromfile</span><span class="p">(</span><span class="n">image_path</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">)</span>

        <span class="n">label</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">]</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">==</span> <span class="s2">&quot;Segmentation&quot;</span><span class="p">:</span>
            <span class="n">segmentation_path</span> <span class="o">=</span> \
                <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="s2">&quot;segmim&quot;</span><span class="p">,</span> <span class="s2">&quot;segmim_&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">[</span><span class="n">index</span><span class="p">])</span><span class="o">.</span><span class="n">zfill</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;.jpg&quot;</span><span class="p">)</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">segmentation_path</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;Can not find segmentation file: &quot;</span> <span class="o">+</span> <span class="n">segmentation_path</span><span class="p">)</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
                <span class="n">segmentation</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">Image</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">segmentation_path</span><span class="p">)</span><span class="o">.</span><span class="n">convert</span><span class="p">(</span><span class="s2">&quot;RGB&quot;</span><span class="p">))</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">segmentation</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">fromfile</span><span class="p">(</span><span class="n">segmentation_path</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">)</span>
            <span class="k">return</span> <span class="n">image</span><span class="p">,</span> <span class="n">segmentation</span><span class="p">,</span> <span class="n">label</span>

        <span class="k">return</span> <span class="n">image</span><span class="p">,</span> <span class="n">label</span>

    <span class="k">def</span> <span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">)</span>


<div class="viewcode-block" id="Flowers102Dataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.Flowers102Dataset.html#mindspore.dataset.Flowers102Dataset">[docs]</a><span class="k">class</span> <span class="nc">Flowers102Dataset</span><span class="p">(</span><span class="n">GeneratorDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses Flowers102 dataset.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, label]` or three :py:obj:`[image, segmentation, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`segmentation` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is a scalar or a tensor of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        task (str): Specify the &#39;Classification&#39; or &#39;Segmentation&#39; task (default=&#39;Classification&#39;).</span>
<span class="sd">        usage (str): Specify the &#39;train&#39;, &#39;valid&#39;, &#39;test&#39; part or &#39;all&#39; parts of dataset</span>
<span class="sd">            (default=&#39;all&#39;, will read all samples).</span>
<span class="sd">        num_samples (int, optional): The number of samples to be included in the dataset (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of subprocesses used to fetch the dataset in parallel (default=1).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset. Random accessible input is required.</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        decode (bool, optional): Whether or not to decode the images and segmentations after reading (default=False).</span>
<span class="sd">        sampler (Union[Sampler, Iterable], optional): Object used to choose samples from the dataset. Random accessible</span>
<span class="sd">            input is required (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            Random accessible input is required. When this argument is specified, &#39;num_samples&#39; reflects the max</span>
<span class="sd">            sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This argument must be specified only</span>
<span class="sd">            when num_shards is also specified. Random accessible input is required.</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a sampler. &#39;sampler&#39; and &#39;shuffle&#39; are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using &#39;sampler&#39; and &#39;shuffle&#39;</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter &#39;sampler&#39;</span>
<span class="sd">         - Parameter &#39;shuffle&#39;</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; flowers102_dataset_dir = &quot;/path/to/flowers102_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.Flowers102Dataset(dataset_dir=flowers102_dataset_dir,</span>
<span class="sd">        ...                                task=&quot;Classification&quot;,</span>
<span class="sd">        ...                                usage=&quot;all&quot;,</span>
<span class="sd">        ...                                decode=True)</span>

<span class="sd">    About Flowers102 dataset:</span>

<span class="sd">    Flowers102 dataset consists of 102 flower categories.</span>
<span class="sd">    The flowers commonly occur in the United Kingdom.</span>
<span class="sd">    Each class consists of between 40 and 258 images.</span>

<span class="sd">    Here is the original Flowers102 dataset structure.</span>
<span class="sd">    You can unzip the dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── flowes102_dataset_dir</span>
<span class="sd">             ├── imagelabels.mat</span>
<span class="sd">             ├── setid.mat</span>
<span class="sd">             ├── jpg</span>
<span class="sd">                  ├── image_00001.jpg</span>
<span class="sd">                  ├── image_00002.jpg</span>
<span class="sd">                  ├── ...</span>
<span class="sd">             ├── segmim</span>
<span class="sd">                  ├── segmim_00001.jpg</span>
<span class="sd">                  ├── segmim_00002.jpg</span>
<span class="sd">                  ├── ...</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @InProceedings{Nilsback08,</span>
<span class="sd">          author       = &quot;Maria-Elena Nilsback and Andrew Zisserman&quot;,</span>
<span class="sd">          title        = &quot;Automated Flower Classification over a Large Number of Classes&quot;,</span>
<span class="sd">          booktitle    = &quot;Indian Conference on Computer Vision, Graphics and Image Processing&quot;,</span>
<span class="sd">          month        = &quot;Dec&quot;,</span>
<span class="sd">          year         = &quot;2008&quot;,</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_flowers102dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">task</span><span class="o">=</span><span class="s2">&quot;Classification&quot;</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="s2">&quot;all&quot;</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
                 <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">realpath</span><span class="p">(</span><span class="n">dataset_dir</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="s2">&quot;Classification&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
        <span class="n">dataset</span> <span class="o">=</span> <span class="n">_Flowers102Dataset</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">)</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">column_names</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">column_names</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span>
                         <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">)</span>

<div class="viewcode-block" id="Flowers102Dataset.get_class_indexing"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.Flowers102Dataset.html#mindspore.dataset.Flowers102Dataset.get_class_indexing">[docs]</a>    <span class="k">def</span> <span class="nf">get_class_indexing</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Get the class index.</span>

<span class="sd">        Returns:</span>
<span class="sd">            dict, a str-to-int mapping from label name to index.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">class_names</span> <span class="o">=</span> <span class="p">[</span>
            <span class="s2">&quot;pink primrose&quot;</span><span class="p">,</span> <span class="s2">&quot;hard-leaved pocket orchid&quot;</span><span class="p">,</span> <span class="s2">&quot;canterbury bells&quot;</span><span class="p">,</span>
            <span class="s2">&quot;sweet pea&quot;</span><span class="p">,</span> <span class="s2">&quot;english marigold&quot;</span><span class="p">,</span> <span class="s2">&quot;tiger lily&quot;</span><span class="p">,</span> <span class="s2">&quot;moon orchid&quot;</span><span class="p">,</span>
            <span class="s2">&quot;bird of paradise&quot;</span><span class="p">,</span> <span class="s2">&quot;monkshood&quot;</span><span class="p">,</span> <span class="s2">&quot;globe thistle&quot;</span><span class="p">,</span> <span class="s2">&quot;snapdragon&quot;</span><span class="p">,</span>
            <span class="s2">&quot;colt&#39;s foot&quot;</span><span class="p">,</span> <span class="s2">&quot;king protea&quot;</span><span class="p">,</span> <span class="s2">&quot;spear thistle&quot;</span><span class="p">,</span> <span class="s2">&quot;yellow iris&quot;</span><span class="p">,</span>
            <span class="s2">&quot;globe-flower&quot;</span><span class="p">,</span> <span class="s2">&quot;purple coneflower&quot;</span><span class="p">,</span> <span class="s2">&quot;peruvian lily&quot;</span><span class="p">,</span> <span class="s2">&quot;balloon flower&quot;</span><span class="p">,</span>
            <span class="s2">&quot;giant white arum lily&quot;</span><span class="p">,</span> <span class="s2">&quot;fire lily&quot;</span><span class="p">,</span> <span class="s2">&quot;pincushion flower&quot;</span><span class="p">,</span> <span class="s2">&quot;fritillary&quot;</span><span class="p">,</span>
            <span class="s2">&quot;red ginger&quot;</span><span class="p">,</span> <span class="s2">&quot;grape hyacinth&quot;</span><span class="p">,</span> <span class="s2">&quot;corn poppy&quot;</span><span class="p">,</span> <span class="s2">&quot;prince of wales feathers&quot;</span><span class="p">,</span>
            <span class="s2">&quot;stemless gentian&quot;</span><span class="p">,</span> <span class="s2">&quot;artichoke&quot;</span><span class="p">,</span> <span class="s2">&quot;sweet william&quot;</span><span class="p">,</span> <span class="s2">&quot;carnation&quot;</span><span class="p">,</span>
            <span class="s2">&quot;garden phlox&quot;</span><span class="p">,</span> <span class="s2">&quot;love in the mist&quot;</span><span class="p">,</span> <span class="s2">&quot;mexican aster&quot;</span><span class="p">,</span> <span class="s2">&quot;alpine sea holly&quot;</span><span class="p">,</span>
            <span class="s2">&quot;ruby-lipped cattleya&quot;</span><span class="p">,</span> <span class="s2">&quot;cape flower&quot;</span><span class="p">,</span> <span class="s2">&quot;great masterwort&quot;</span><span class="p">,</span> <span class="s2">&quot;siam tulip&quot;</span><span class="p">,</span>
            <span class="s2">&quot;lenten rose&quot;</span><span class="p">,</span> <span class="s2">&quot;barbeton daisy&quot;</span><span class="p">,</span> <span class="s2">&quot;daffodil&quot;</span><span class="p">,</span> <span class="s2">&quot;sword lily&quot;</span><span class="p">,</span> <span class="s2">&quot;poinsettia&quot;</span><span class="p">,</span>
            <span class="s2">&quot;bolero deep blue&quot;</span><span class="p">,</span> <span class="s2">&quot;wallflower&quot;</span><span class="p">,</span> <span class="s2">&quot;marigold&quot;</span><span class="p">,</span> <span class="s2">&quot;buttercup&quot;</span><span class="p">,</span> <span class="s2">&quot;oxeye daisy&quot;</span><span class="p">,</span>
            <span class="s2">&quot;common dandelion&quot;</span><span class="p">,</span> <span class="s2">&quot;petunia&quot;</span><span class="p">,</span> <span class="s2">&quot;wild pansy&quot;</span><span class="p">,</span> <span class="s2">&quot;primula&quot;</span><span class="p">,</span> <span class="s2">&quot;sunflower&quot;</span><span class="p">,</span>
            <span class="s2">&quot;pelargonium&quot;</span><span class="p">,</span> <span class="s2">&quot;bishop of llandaff&quot;</span><span class="p">,</span> <span class="s2">&quot;gaura&quot;</span><span class="p">,</span> <span class="s2">&quot;geranium&quot;</span><span class="p">,</span> <span class="s2">&quot;orange dahlia&quot;</span><span class="p">,</span>
            <span class="s2">&quot;pink-yellow dahlia?&quot;</span><span class="p">,</span> <span class="s2">&quot;cautleya spicata&quot;</span><span class="p">,</span> <span class="s2">&quot;japanese anemone&quot;</span><span class="p">,</span>
            <span class="s2">&quot;black-eyed susan&quot;</span><span class="p">,</span> <span class="s2">&quot;silverbush&quot;</span><span class="p">,</span> <span class="s2">&quot;californian poppy&quot;</span><span class="p">,</span> <span class="s2">&quot;osteospermum&quot;</span><span class="p">,</span>
            <span class="s2">&quot;spring crocus&quot;</span><span class="p">,</span> <span class="s2">&quot;bearded iris&quot;</span><span class="p">,</span> <span class="s2">&quot;windflower&quot;</span><span class="p">,</span> <span class="s2">&quot;tree poppy&quot;</span><span class="p">,</span> <span class="s2">&quot;gazania&quot;</span><span class="p">,</span>
            <span class="s2">&quot;azalea&quot;</span><span class="p">,</span> <span class="s2">&quot;water lily&quot;</span><span class="p">,</span> <span class="s2">&quot;rose&quot;</span><span class="p">,</span> <span class="s2">&quot;thorn apple&quot;</span><span class="p">,</span> <span class="s2">&quot;morning glory&quot;</span><span class="p">,</span>
            <span class="s2">&quot;passion flower&quot;</span><span class="p">,</span> <span class="s2">&quot;lotus&quot;</span><span class="p">,</span> <span class="s2">&quot;toad lily&quot;</span><span class="p">,</span> <span class="s2">&quot;anthurium&quot;</span><span class="p">,</span> <span class="s2">&quot;frangipani&quot;</span><span class="p">,</span>
            <span class="s2">&quot;clematis&quot;</span><span class="p">,</span> <span class="s2">&quot;hibiscus&quot;</span><span class="p">,</span> <span class="s2">&quot;columbine&quot;</span><span class="p">,</span> <span class="s2">&quot;desert-rose&quot;</span><span class="p">,</span> <span class="s2">&quot;tree mallow&quot;</span><span class="p">,</span>
            <span class="s2">&quot;magnolia&quot;</span><span class="p">,</span> <span class="s2">&quot;cyclamen&quot;</span><span class="p">,</span> <span class="s2">&quot;watercress&quot;</span><span class="p">,</span> <span class="s2">&quot;canna lily&quot;</span><span class="p">,</span> <span class="s2">&quot;hippeastrum&quot;</span><span class="p">,</span>
            <span class="s2">&quot;bee balm&quot;</span><span class="p">,</span> <span class="s2">&quot;ball moss&quot;</span><span class="p">,</span> <span class="s2">&quot;foxglove&quot;</span><span class="p">,</span> <span class="s2">&quot;bougainvillea&quot;</span><span class="p">,</span> <span class="s2">&quot;camellia&quot;</span><span class="p">,</span> <span class="s2">&quot;mallow&quot;</span><span class="p">,</span>
            <span class="s2">&quot;mexican petunia&quot;</span><span class="p">,</span> <span class="s2">&quot;bromelia&quot;</span><span class="p">,</span> <span class="s2">&quot;blanket flower&quot;</span><span class="p">,</span> <span class="s2">&quot;trumpet creeper&quot;</span><span class="p">,</span>
            <span class="s2">&quot;blackberry lily&quot;</span>
        <span class="p">]</span>

        <span class="n">class_dict</span> <span class="o">=</span> <span class="p">{}</span>
        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">class_name</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">class_names</span><span class="p">):</span>
            <span class="n">class_dict</span><span class="p">[</span><span class="n">class_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">i</span>

        <span class="k">return</span> <span class="n">class_dict</span></div></div>


<span class="k">class</span> <span class="nc">ImageFolderDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads images from a tree of directories.</span>
<span class="sd">    All images within one folder have the same label.</span>

<span class="sd">    The generated dataset has two columns: :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is of a scalar of uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        extensions (list[str], optional): List of file extensions to be</span>
<span class="sd">            included in the dataset (default=None).</span>
<span class="sd">        class_indexing (dict, optional): A str-to-int mapping from folder name to index</span>
<span class="sd">            (default=None, the folder names will be sorted</span>
<span class="sd">            alphabetically and each class will be given a</span>
<span class="sd">            unique index starting from 0).</span>
<span class="sd">        decode (bool, optional): Decode the images after reading (default=False).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        RuntimeError: If class_indexing is not a dictionary.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - The shape of the image column is [image_size] if decode flag is False, or [H,W,C] otherwise.</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; image_folder_dataset_dir = &quot;/path/to/image_folder_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Read all samples (image files) in image_folder_dataset_dir with 8 threads</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.ImageFolderDataset(dataset_dir=image_folder_dataset_dir,</span>
<span class="sd">        ...                                 num_parallel_workers=8)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Read all samples (image files) from folder cat and folder dog with label 0 and 1</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.ImageFolderDataset(dataset_dir=image_folder_dataset_dir,</span>
<span class="sd">        ...                                 class_indexing={&quot;cat&quot;:0, &quot;dog&quot;:1})</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 3) Read all samples (image files) in image_folder_dataset_dir with extensions .JPEG</span>
<span class="sd">        &gt;&gt;&gt; #    and .png (case sensitive)</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.ImageFolderDataset(dataset_dir=image_folder_dataset_dir,</span>
<span class="sd">        ...                                 extensions=[&quot;.JPEG&quot;, &quot;.png&quot;])</span>

<span class="sd">    About ImageFolderDataset:</span>

<span class="sd">    You can construct the following directory structure from your dataset files and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── image_folder_dataset_directory</span>
<span class="sd">             ├── class1</span>
<span class="sd">             │    ├── 000000000001.jpg</span>
<span class="sd">             │    ├── 000000000002.jpg</span>
<span class="sd">             │    ├── ...</span>
<span class="sd">             ├── class2</span>
<span class="sd">             │    ├── 000000000001.jpg</span>
<span class="sd">             │    ├── 000000000002.jpg</span>
<span class="sd">             │    ├── ...</span>
<span class="sd">             ├── class3</span>
<span class="sd">             │    ├── 000000000001.jpg</span>
<span class="sd">             │    ├── 000000000002.jpg</span>
<span class="sd">             │    ├── ...</span>
<span class="sd">             ├── classN</span>
<span class="sd">             ├── ...</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_imagefolderdataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">extensions</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">class_indexing</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">extensions</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">extensions</span><span class="p">,</span> <span class="p">[])</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">class_indexing</span><span class="p">,</span> <span class="p">{})</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">ImageFolderNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">extensions</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span><span class="p">)</span>


<div class="viewcode-block" id="KMnistDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.KMnistDataset.html#mindspore.dataset.KMnistDataset">[docs]</a><span class="k">class</span> <span class="nc">KMnistDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses the KMNIST dataset.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is a scalar of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str, optional): Usage of this dataset, can be `train`, `test` or `all` . `train` will read from 60,000</span>
<span class="sd">            train samples, `test` will read from 10,000 test samples, `all` will read from all 70,000 samples.</span>
<span class="sd">            (default=None, will read all samples)</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, will read all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, will use value set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            When this argument is specified, `num_samples` reflects the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This</span>
<span class="sd">            argument can only be specified when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If `dataset_dir` does not contain data files.</span>
<span class="sd">        RuntimeError: If `num_parallel_workers` exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If `sampler` and `shuffle` are specified at the same time.</span>
<span class="sd">        RuntimeError: If `sampler` and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If `num_shards` is specified but `shard_id` is None.</span>
<span class="sd">        RuntimeError: If `shard_id` is specified but `num_shards` is None.</span>
<span class="sd">        ValueError: If `shard_id` is invalid (out of range [0, `num_shards`]).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; kmnist_dataset_dir = &quot;/path/to/kmnist_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Read 3 samples from KMNIST dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.KMnistDataset(dataset_dir=kmnist_dataset_dir, num_samples=3)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Note: In kmnist_dataset dataset, each dictionary has keys &quot;image&quot; and &quot;label&quot;</span>

<span class="sd">    About KMNIST dataset:</span>

<span class="sd">    KMNIST is a dataset, adapted from Kuzushiji Dataset, as a drop-in replacement for MNIST dataset,</span>
<span class="sd">    which is the most famous dataset in the machine learning community.</span>

<span class="sd">    Here is the original KMNIST dataset structure.</span>
<span class="sd">    You can unzip the dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── kmnist_dataset_dir</span>
<span class="sd">             ├── t10k-images-idx3-ubyte</span>
<span class="sd">             ├── t10k-labels-idx1-ubyte</span>
<span class="sd">             ├── train-images-idx3-ubyte</span>
<span class="sd">             └── train-labels-idx1-ubyte</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @online{clanuwat2018deep,</span>
<span class="sd">          author       = {Tarin Clanuwat and Mikel Bober-Irizar and Asanobu Kitamoto and</span>
<span class="sd">                           Alex Lamb and Kazuaki Yamamoto and David Ha},</span>
<span class="sd">          title        = {Deep Learning for Classical Japanese Literature},</span>
<span class="sd">          date         = {2018-12-03},</span>
<span class="sd">          year         = {2018},</span>
<span class="sd">          eprintclass  = {cs.CV},</span>
<span class="sd">          eprinttype   = {arXiv},</span>
<span class="sd">          eprint       = {cs.CV/1812.01718},</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_mnist_cifar_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">KMnistNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span></div>


<span class="k">class</span> <span class="nc">ManifestDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset for reading images from a Manifest file.</span>

<span class="sd">    The generated dataset has two columns: :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is of a scalar of uint64 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_file (str): File to be read.</span>
<span class="sd">        usage (str, optional): Acceptable usages include `train`, `eval` and `inference` (default= `train`).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset.</span>
<span class="sd">            (default=None, will include all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, will use value set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected</span>
<span class="sd">            order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        class_indexing (dict, optional): A str-to-int mapping from label name to index</span>
<span class="sd">            (default=None, the folder names will be sorted alphabetically and each</span>
<span class="sd">            class will be given a unique index starting from 0).</span>
<span class="sd">        decode (bool, optional): decode the images after reading (default=False).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the max number of samples per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This</span>
<span class="sd">            argument can only be specified when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_files are not valid or do not exist.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        RuntimeError: If class_indexing is not a dictionary.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - The shape of the image column is [image_size] if decode flag is False, or [H,W,C] otherwise.</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; manifest_dataset_dir = &quot;/path/to/manifest_dataset_file&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Read all samples specified in manifest_dataset_dir dataset with 8 threads for training</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.ManifestDataset(dataset_file=manifest_dataset_dir, usage=&quot;train&quot;, num_parallel_workers=8)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Read samples (specified in manifest_file.manifest) for shard 0 in a 2-way distributed training setup</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.ManifestDataset(dataset_file=manifest_dataset_dir, num_shards=2, shard_id=0)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_manifestdataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_file</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="s2">&quot;train&quot;</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">class_indexing</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_file</span> <span class="o">=</span> <span class="n">dataset_file</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;train&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">class_indexing</span><span class="p">,</span> <span class="p">{})</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">ManifestNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_file</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">get_class_indexing</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Get the class index.</span>

<span class="sd">        Returns:</span>
<span class="sd">            dict, a str-to-int mapping from label name to index.</span>

<span class="sd">        Examples:</span>
<span class="sd">            &gt;&gt;&gt; manifest_dataset_dir = &quot;/path/to/manifest_dataset_file&quot;</span>
<span class="sd">            &gt;&gt;&gt;</span>
<span class="sd">            &gt;&gt;&gt; dataset = ds.ManifestDataset(dataset_file=manifest_dataset_dir)</span>
<span class="sd">            &gt;&gt;&gt; class_indexing = dataset.get_class_indexing()</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_class_indexing</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">runtime_getter</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_init_tree_getters</span><span class="p">()</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_class_indexing</span> <span class="o">=</span> <span class="n">runtime_getter</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">GetClassIndexing</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span> <span class="o">=</span> <span class="p">{}</span>
            <span class="k">for</span> <span class="n">pair</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_class_indexing</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span><span class="p">[</span><span class="n">pair</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span> <span class="o">=</span> <span class="n">pair</span><span class="p">[</span><span class="mi">1</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span>


<span class="k">class</span> <span class="nc">MnistDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses the MNIST dataset.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is a scalar of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str, optional): Usage of this dataset, can be `train`, `test` or `all` . `train` will read from 60,000</span>
<span class="sd">            train samples, `test` will read from 10,000 test samples, `all` will read from all 70,000 samples.</span>
<span class="sd">            (default=None, will read all samples)</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, will read all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, will use value set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            When this argument is specified, `num_samples` reflects the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This</span>
<span class="sd">            argument can only be specified when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; mnist_dataset_dir = &quot;/path/to/mnist_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Read 3 samples from MNIST dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.MnistDataset(dataset_dir=mnist_dataset_dir, num_samples=3)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Note: In mnist_dataset dataset, each dictionary has keys &quot;image&quot; and &quot;label&quot;</span>

<span class="sd">    About MNIST dataset:</span>

<span class="sd">    The MNIST database of handwritten digits has a training set of 60,000 examples,</span>
<span class="sd">    and a test set of 10,000 examples. It is a subset of a larger set available from</span>
<span class="sd">    NIST. The digits have been size-normalized and centered in a fixed-size image.</span>

<span class="sd">    Here is the original MNIST dataset structure.</span>
<span class="sd">    You can unzip the dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── mnist_dataset_dir</span>
<span class="sd">             ├── t10k-images-idx3-ubyte</span>
<span class="sd">             ├── t10k-labels-idx1-ubyte</span>
<span class="sd">             ├── train-images-idx3-ubyte</span>
<span class="sd">             └── train-labels-idx1-ubyte</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @article{lecun2010mnist,</span>
<span class="sd">        title        = {MNIST handwritten digit database},</span>
<span class="sd">        author       = {LeCun, Yann and Cortes, Corinna and Burges, CJ},</span>
<span class="sd">        journal      = {ATT Labs [Online]},</span>
<span class="sd">        volume       = {2},</span>
<span class="sd">        year         = {2010},</span>
<span class="sd">        howpublished = {http://yann.lecun.com/exdb/mnist}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_mnist_cifar_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">MnistNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span>


<div class="viewcode-block" id="PhotoTourDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.PhotoTourDataset.html#mindspore.dataset.PhotoTourDataset">[docs]</a><span class="k">class</span> <span class="nc">PhotoTourDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses the PhotoTour dataset.</span>

<span class="sd">    The generated dataset with different usage has different output columns.</span>
<span class="sd">    If train, the generated dataset has one column :py:obj:`[image]`,</span>
<span class="sd">    else three columns :py:obj:`[image1, image2, matches]`.</span>
<span class="sd">    The tensor of column :py:obj:`image`, :py:obj:`image1` and :py:obj:`image2` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`matches` is a scalar of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        name (str): Name of the dataset to load,</span>
<span class="sd">            should be one of &#39;notredame&#39;, &#39;yosemite&#39;, &#39;liberty&#39;, &#39;notredame_harris&#39;,</span>
<span class="sd">            &#39;yosemite_harris&#39; or &#39;liberty_harris&#39;.</span>
<span class="sd">        usage (str, optional): Usage of the dataset, can be `train` or `test` (Default=None, will be set to &#39;train&#39;).</span>
<span class="sd">            When usage is `train`, number of samples for each `name` is</span>
<span class="sd">            {&#39;notredame&#39;: 468159, &#39;yosemite&#39;: 633587, &#39;liberty&#39;: 450092, &#39;liberty_harris&#39;: 379587,</span>
<span class="sd">            &#39;yosemite_harris&#39;: 450912, &#39;notredame_harris&#39;: 325295}.</span>
<span class="sd">            When usage is `test`, will read 100,000 samples for testing.</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, will read all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, will use value set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            When this argument is specified, `num_samples` reflects the max sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This</span>
<span class="sd">            argument can only be specified when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If dataset_dir is not exist.</span>
<span class="sd">        ValueError: If usage is not in [&quot;train&quot;, &quot;test&quot;].</span>
<span class="sd">        ValueError: If name is not in [&quot;notredame&quot;, &quot;yosemite&quot;, &quot;liberty&quot;,</span>
<span class="sd">            &quot;notredame_harris&quot;, &quot;yosemite_harris&quot;, &quot;liberty_harris&quot;].</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a sampler. `sampler` and `shuffle` are mutually exclusive. The table</span>
<span class="sd">          below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using &#39;sampler&#39; and &#39;shuffle&#39;</span>
<span class="sd">       :widths: 64 64 1</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; # Read 3 samples from PhotoTour dataset.</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.PhotoTourDataset(dataset_dir=&quot;/path/to/photo_tour_dataset_directory&quot;,</span>
<span class="sd">        ...                               name=&#39;liberty&#39;, usage=&#39;train&#39;, num_samples=3)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # In PhotoTourDataset dataset, if usage is &#39;train&#39;, each dictionary has key &quot;image&quot;,</span>
<span class="sd">        &gt;&gt;&gt; # else has keys &quot;image1&quot; &quot;image2&quot; and &quot;matches&quot;.</span>

<span class="sd">    About PhotoTour dataset:</span>

<span class="sd">    The data is taken from Photo Tourism reconstructions from Trevi Fountain (Rome), Notre Dame (Paris) and Half</span>
<span class="sd">    Dome (Yosemite). Each dataset consists of a series of corresponding patches, which are obtained by projecting</span>
<span class="sd">    3D points from Photo Tourism reconstructions back into the original images.</span>

<span class="sd">    The dataset consists of 1024 x 1024 bitmap (.bmp) images, each containing a 16 x 16 array of image patches.</span>
<span class="sd">    Each patch is sampled as 64 x 64 grayscale, with a canonical scale and orientation. For details of how the scale</span>
<span class="sd">    and orientation is established, please see the paper. An associated metadata file info.txt contains the match</span>
<span class="sd">    information. Each row of info.txt corresponds to a separate patch, with the patches ordered from left to right and</span>
<span class="sd">    top to bottom in each bitmap image. The first number on each row of info.txt is the 3D point ID from which that</span>
<span class="sd">    patch was sampled -- patches with the same 3D point ID are projected from the same 3D point (into different images).</span>
<span class="sd">    The second number in info.txt corresponds to the image from which the patch was sampled, and is not used at present.</span>

<span class="sd">    You can unzip the original PhotoTour dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── photo_tour_dataset_directory</span>
<span class="sd">            ├── liberty/</span>
<span class="sd">            │    ├── info.txt                 // two columns: 3D_point_ID, unused</span>
<span class="sd">            │    ├── m50_100000_100000_0.txt  // seven columns: patch_ID1, 3D_point_ID1, unused1,</span>
<span class="sd">            │    │                            // patch_ID2, 3D_point_ID2, unused2, unused3</span>
<span class="sd">            │    ├── patches0000.bmp          // 1024*1024 pixels, with 16 * 16 patches.</span>
<span class="sd">            │    ├── patches0001.bmp</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            ├── yosemite/</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            ├── notredame/</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            ├── liberty_harris/</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            ├── yosemite_harris/</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            ├── notredame_harris/</span>
<span class="sd">            │    ├── ...</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @INPROCEEDINGS{4269996,</span>
<span class="sd">            author={Winder, Simon A. J. and Brown, Matthew},</span>
<span class="sd">            booktitle={2007 IEEE Conference on Computer Vision and Pattern Recognition},</span>
<span class="sd">            title={Learning Local Image Descriptors},</span>
<span class="sd">            year={2007},</span>
<span class="sd">            volume={},</span>
<span class="sd">            number={},</span>
<span class="sd">            pages={1-8},</span>
<span class="sd">            doi={10.1109/CVPR.2007.382971}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_photo_tour_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">name</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;train&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">PhotoTourNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span></div>


<div class="viewcode-block" id="Places365Dataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.Places365Dataset.html#mindspore.dataset.Places365Dataset">[docs]</a><span class="k">class</span> <span class="nc">Places365Dataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses the Places365 dataset.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is a scalar of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str, optional): Usage of this dataset, can be `train-standard`, `train-challenge` or `val`</span>
<span class="sd">            (default=None, will be set to &#39;train-standard&#39;).</span>
<span class="sd">        small (bool, optional): Use 256 * 256 images (True) or high resolution images (False) (default=False).</span>
<span class="sd">        decode (bool, optional): Decode the images after reading (default=True).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, will read all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, will use value set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            When this argument is specified, `num_samples` reflects the max sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This</span>
<span class="sd">            argument can only be specified when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>
<span class="sd">        ValueError: If usage is not in [&quot;train-standard&quot;, &quot;train-challenge&quot;, &quot;val&quot;].</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a sampler. &#39;sampler&#39; and &#39;shuffle&#39; are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using &#39;sampler&#39; and &#39;shuffle&#39;</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; place365_dataset_dir = &quot;/path/to/place365_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Read 3 samples from Places365 dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.Places365Dataset(dataset_dir=place365_dataset_dir, usage=&#39;train-standard&#39;,</span>
<span class="sd">        ...                               small=True, decode=True, num_samples=3)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # In places365 dataset, each dictionary has keys &quot;image&quot; and &quot;label&quot;.</span>

<span class="sd">    About Places365 dataset:</span>

<span class="sd">    Convolutional neural networks (CNNs) trained on the Places2 Database can be used for scene recognition as well as</span>
<span class="sd">    generic deep scene features for visual recognition.</span>

<span class="sd">    The author releases the data of Places365-Standard and the data of Places365-Challenge to the public.</span>
<span class="sd">    Places365-Standard is the core set of Places2 Database, which has been used to train the Places365-CNNs. The author</span>
<span class="sd">    will add other kinds of annotation on the Places365-Standard in the future. Places365-Challenge is the competition</span>
<span class="sd">    set of Places2 Database, which has 6.2 million extra images compared to the Places365-Standard.</span>
<span class="sd">    The Places365-Challenge will be used for the Places Challenge 2016.</span>

<span class="sd">    You can unzip the original Places365 dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── categories_places365</span>
<span class="sd">            ├── places365_train-standard.txt</span>
<span class="sd">            ├── places365_train-challenge.txt</span>
<span class="sd">            ├── val_large/</span>
<span class="sd">            │    ├── Places365_val_00000001.jpg</span>
<span class="sd">            │    ├── Places365_val_00000002.jpg</span>
<span class="sd">            │    ├── Places365_val_00000003.jpg</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            ├── val_256/</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            ├── data_large_standard/</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            ├── data_256_standard/</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            ├── data_large_challenge/</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            ├── data_256_challenge /</span>
<span class="sd">            │    ├── ...</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        article{zhou2017places,</span>
<span class="sd">            title={Places: A 10 million Image Database for Scene Recognition},</span>
<span class="sd">            author={Zhou, Bolei and Lapedriza, Agata and Khosla, Aditya and Oliva, Aude and Torralba, Antonio},</span>
<span class="sd">            journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},</span>
<span class="sd">            year={2017},</span>
<span class="sd">            publisher={IEEE}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_places365_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">small</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">dataset_dir</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;train-standard&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">small</span> <span class="o">=</span> <span class="n">small</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">decode</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">Places365Node</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">small</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span></div>


<div class="viewcode-block" id="QMnistDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.QMnistDataset.html#mindspore.dataset.QMnistDataset">[docs]</a><span class="k">class</span> <span class="nc">QMnistDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses the QMNIST dataset.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is a scalar when `compat` is True else a tensor both of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str, optional): Usage of this dataset, can be `train`, `test`, `test10k`, `test50k`, `nist`</span>
<span class="sd">            or `all` (default=None, will read all samples).</span>
<span class="sd">        compat (bool, optional): Whether the label for each example is class number (compat=True) or the full QMNIST</span>
<span class="sd">            information (compat=False) (default=True).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, will read all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, will use value set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            When this argument is specified, `num_samples` reflects the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This</span>
<span class="sd">            argument can only be specified when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; qmnist_dataset_dir = &quot;/path/to/qmnist_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Read 3 samples from QMNIST train dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.QMnistDataset(dataset_dir=qmnist_dataset_dir, num_samples=3)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Note: In QMNIST dataset, each dictionary has keys &quot;image&quot; and &quot;label&quot;</span>

<span class="sd">    About QMNIST dataset:</span>

<span class="sd">    The QMNIST dataset was generated from the original data found in the NIST Special Database 19 with the goal to</span>
<span class="sd">    match the MNIST preprocessing as closely as possible.</span>
<span class="sd">    Through an iterative process, researchers tried to generate an additional 50k images of MNIST-like data.</span>
<span class="sd">    They started with a reconstruction process given in the paper and used the Hungarian algorithm to find the best</span>
<span class="sd">    matches between the original MNIST samples and their reconstructed samples.</span>

<span class="sd">    Here is the original QMNIST dataset structure.</span>
<span class="sd">    You can unzip the dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── qmnist_dataset_dir</span>
<span class="sd">             ├── qmnist-train-images-idx3-ubyte</span>
<span class="sd">             ├── qmnist-train-labels-idx2-int</span>
<span class="sd">             ├── qmnist-test-images-idx3-ubyte</span>
<span class="sd">             ├── qmnist-test-labels-idx2-int</span>
<span class="sd">             ├── xnist-images-idx3-ubyte</span>
<span class="sd">             └── xnist-labels-idx2-int</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @incollection{qmnist-2019,</span>
<span class="sd">           title = &quot;Cold Case: The Lost MNIST Digits&quot;,</span>
<span class="sd">           author = &quot;Chhavi Yadav and L\&#39;{e}on Bottou&quot;,\</span>
<span class="sd">           booktitle = {Advances in Neural Information Processing Systems 32},</span>
<span class="sd">           year = {2019},</span>
<span class="sd">           publisher = {Curran Associates, Inc.},</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_qmnist_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">compat</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">compat</span> <span class="o">=</span> <span class="n">compat</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">QMnistNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">compat</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span></div>


<div class="viewcode-block" id="RandomDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.RandomDataset.html#mindspore.dataset.RandomDataset">[docs]</a><span class="k">class</span> <span class="nc">RandomDataset</span><span class="p">(</span><span class="n">SourceDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that generates random data.</span>

<span class="sd">    Args:</span>
<span class="sd">        total_rows (int, optional): Number of samples for the dataset to generate</span>
<span class="sd">            (default=None, number of samples is random).</span>
<span class="sd">        schema (Union[str, Schema], optional): Path to the JSON schema file or schema object (default=None).</span>
<span class="sd">            If the schema is not provided, the random dataset generates a random schema.</span>
<span class="sd">        columns_list (list[str], optional): List of columns to be read (default=None, read all columns)</span>
<span class="sd">        num_samples (int, optional): The number of samples to be included in the dataset</span>
<span class="sd">            (default=None, all samples).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, number set in the config).</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, &#39;num_samples&#39; reflects</span>
<span class="sd">            the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_random_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">total_rows</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">columns_list</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span>
                         <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">total_rows</span> <span class="o">=</span> <span class="n">total_rows</span>
        <span class="k">if</span> <span class="n">schema</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">total_rows</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">total_rows</span><span class="p">,</span> <span class="n">Schema</span><span class="o">.</span><span class="n">get_num_rows</span><span class="p">(</span><span class="n">schema</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">columns_list</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">columns_list</span><span class="p">,</span> <span class="p">[])</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="n">schema</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">cpp_schema</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="n">Schema</span><span class="p">)</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">RandomNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">total_rows</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">columns_list</span><span class="p">)</span></div>


<span class="k">class</span> <span class="nc">_SBDataset</span><span class="p">:</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Dealing with the data file with .mat extension, and return one row in tuple (image, task) each time.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="n">usage</span><span class="p">,</span> <span class="n">decode</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">column_list</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;image&#39;</span><span class="p">,</span> <span class="s1">&#39;task&#39;</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">task</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">images_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="s1">&#39;img&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">cls_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="s1">&#39;cls&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_loadmat</span> <span class="o">=</span> <span class="n">loadmat</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">categories</span> <span class="o">=</span> <span class="mi">20</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">usage</span> <span class="o">==</span> <span class="s2">&quot;all&quot;</span><span class="p">:</span>
            <span class="n">image_names</span> <span class="o">=</span> <span class="p">[]</span>
            <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;train&quot;</span><span class="p">,</span> <span class="s2">&quot;val&quot;</span><span class="p">]:</span>
                <span class="n">usage_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="n">item</span> <span class="o">+</span> <span class="s1">&#39;.txt&#39;</span><span class="p">)</span>
                <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">usage_path</span><span class="p">):</span>
                    <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span><span class="s2">&quot;SBDataset: </span><span class="si">{0}</span><span class="s2"> not found&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">usage_path</span><span class="p">))</span>
                <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">usage_path</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
                    <span class="n">image_names</span> <span class="o">+=</span> <span class="p">[</span><span class="n">x</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">f</span><span class="o">.</span><span class="n">readlines</span><span class="p">()]</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">usage_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span> <span class="o">+</span> <span class="s1">&#39;.txt&#39;</span><span class="p">)</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">usage_path</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span><span class="s2">&quot;SBDataset: </span><span class="si">{0}</span><span class="s2"> not found&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">usage_path</span><span class="p">))</span>
            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">usage_path</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
                <span class="n">image_names</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">f</span><span class="o">.</span><span class="n">readlines</span><span class="p">()]</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">images</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">images_path</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="s2">&quot;.jpg&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">image_names</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">clss</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cls_path</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="s2">&quot;.mat&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">image_names</span><span class="p">]</span>

        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">images</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">clss</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;SBDataset: images count not equal to cls count&quot;</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_get_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_boundaries_data</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">==</span> <span class="s2">&quot;Boundaries&quot;</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_segmentation_data</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_get_item</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_decode_item</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_undecode_item</span>

    <span class="k">def</span> <span class="nf">_get_boundaries_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mat_path</span><span class="p">):</span>
        <span class="n">mat_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_loadmat</span><span class="p">(</span><span class="n">mat_path</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">np</span><span class="o">.</span><span class="n">expand_dims</span><span class="p">(</span><span class="n">mat_data</span><span class="p">[</span><span class="s1">&#39;GTcls&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="p">][</span><span class="mi">0</span><span class="p">][</span><span class="n">i</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">toarray</span><span class="p">(),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
                               <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">categories</span><span class="p">)],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_get_segmentation_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mat_path</span><span class="p">):</span>
        <span class="n">mat_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_loadmat</span><span class="p">(</span><span class="n">mat_path</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">Image</span><span class="o">.</span><span class="n">fromarray</span><span class="p">(</span><span class="n">mat_data</span><span class="p">[</span><span class="s1">&#39;GTcls&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">][</span><span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="p">][</span><span class="mi">0</span><span class="p">])</span>

    <span class="k">def</span> <span class="nf">_get_decode_item</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">idx</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">Image</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">images</span><span class="p">[</span><span class="n">idx</span><span class="p">])</span><span class="o">.</span><span class="n">convert</span><span class="p">(</span><span class="s1">&#39;RGB&#39;</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_data</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">clss</span><span class="p">[</span><span class="n">idx</span><span class="p">])</span>

    <span class="k">def</span> <span class="nf">_get_undecode_item</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">idx</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">fromfile</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">images</span><span class="p">[</span><span class="n">idx</span><span class="p">],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_data</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">clss</span><span class="p">[</span><span class="n">idx</span><span class="p">])</span>

    <span class="k">def</span> <span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">images</span><span class="p">)</span>

    <span class="k">def</span> <span class="fm">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">idx</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_item</span><span class="p">(</span><span class="n">idx</span><span class="p">)</span>


<div class="viewcode-block" id="SBDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.SBDataset.html#mindspore.dataset.SBDataset">[docs]</a><span class="k">class</span> <span class="nc">SBDataset</span><span class="p">(</span><span class="n">GeneratorDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses Semantic Boundaries Dataset.</span>

<span class="sd">    The generated dataset has two columns: :py:obj:`[image, task]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`task` contains 20 images of the uint8 type if `task` is `Boundaries` otherwise</span>
<span class="sd">    contains 1 image of the uint8 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        task (str, optional): Acceptable tasks include `Boundaries` or `Segmentation` (default= `Boundaries`).</span>
<span class="sd">        usage (str, optional): Acceptable usages include `train`, `val`, `train_noval` and `all` (default= `all`).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset.</span>
<span class="sd">            (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, number set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected</span>
<span class="sd">            order behavior shown in the table).</span>
<span class="sd">        decode (bool, optional): Decode the images after reading (default=None).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the max sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir is not valid or does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If dataset_dir is not exist.</span>
<span class="sd">        ValueError: If task is not in [`Boundaries`, `Segmentation`].</span>
<span class="sd">        ValueError: If usage is not in [`train`, `val`, `train_noval`, `all`].</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a sampler. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; sb_dataset_dir = &quot;/path/to/sb_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Get all samples from Semantic Boundaries Dataset in sequence</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.SBDataset(dataset_dir=sb_dataset_dir, shuffle=False)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Randomly select 350 samples from Semantic Boundaries Dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.SBDataset(dataset_dir=sb_dataset_dir, num_samples=350, shuffle=True)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 3) Get samples from Semantic Boundaries Dataset for shard 0 in a 2-way distributed training</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.SBDataset(dataset_dir=sb_dataset_dir, num_shards=2, shard_id=0)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # In Semantic Boundaries Dataset, each dictionary has keys &quot;image&quot; and &quot;task&quot;</span>

<span class="sd">    About Semantic Boundaries Dataset:</span>

<span class="sd">    The Semantic Boundaries Dataset consists of 11355 colour images. There are 8498 images&#39; name in the train.txt,</span>
<span class="sd">    2857 images&#39; name in the val.txt and 5623 images&#39; name in the train_noval.txt. The category cls/</span>
<span class="sd">    contains the Segmentation and Boundaries results of category-level, the category inst/ catains the</span>
<span class="sd">    Segmentation and Boundaries results of instance-level.</span>

<span class="sd">    You can unzip the dataset files into the following structure and read by MindSpore&#39;s API:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">         .</span>
<span class="sd">         └── benchmark_RELEASE</span>
<span class="sd">              ├── dataset</span>
<span class="sd">              ├── img</span>
<span class="sd">              │    ├── 2008_000002.jpg</span>
<span class="sd">              │    ├── 2008_000003.jpg</span>
<span class="sd">              │    ├── ...</span>
<span class="sd">              ├── cls</span>
<span class="sd">              │    ├── 2008_000002.mat</span>
<span class="sd">              │    ├── 2008_000003.mat</span>
<span class="sd">              │    ├── ...</span>
<span class="sd">              ├── inst</span>
<span class="sd">              │    ├── 2008_000002.mat</span>
<span class="sd">              │    ├── 2008_000003.mat</span>
<span class="sd">              │    ├── ...</span>
<span class="sd">              ├── train.txt</span>
<span class="sd">              └── val.txt</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @InProceedings{BharathICCV2011,</span>
<span class="sd">            author       = &quot;Bharath Hariharan and Pablo Arbelaez and Lubomir Bourdev and</span>
<span class="sd">                            Subhransu Maji and Jitendra Malik&quot;,</span>
<span class="sd">            title        = &quot;Semantic Contours from Inverse Detectors&quot;,</span>
<span class="sd">            booktitle    = &quot;International Conference on Computer Vision (ICCV)&quot;,</span>
<span class="sd">            year         = &quot;2011&quot;,</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_sb_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">task</span><span class="o">=</span><span class="s1">&#39;Boundaries&#39;</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="s1">&#39;all&#39;</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
                 <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="n">dataset</span> <span class="o">=</span> <span class="n">_SBDataset</span><span class="p">(</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="n">usage</span><span class="p">,</span> <span class="n">decode</span><span class="p">)</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">column_names</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">column_list</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span>
                         <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">)</span></div>


<div class="viewcode-block" id="SBUDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.SBUDataset.html#mindspore.dataset.SBUDataset">[docs]</a><span class="k">class</span> <span class="nc">SBUDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses the SBU dataset.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, caption]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`caption` is of the string type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        decode (bool, optional): Decode the images after reading (default=False).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, will read all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, will use value set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            When this argument is specified, `num_samples` reflects the max sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This</span>
<span class="sd">            argument can only be specified when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a sampler. &#39;sampler&#39; and &#39;shuffle&#39; are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using &#39;sampler&#39; and &#39;shuffle&#39;</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter &#39;sampler&#39;</span>
<span class="sd">         - Parameter &#39;shuffle&#39;</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; sbu_dataset_dir = &quot;/path/to/sbu_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt; # Read 3 samples from SBU dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.SBUDataset(dataset_dir=sbu_dataset_dir, num_samples=3)</span>

<span class="sd">    About SBU dataset:</span>

<span class="sd">    SBU dataset is a large captioned photo collection.</span>
<span class="sd">    It contains one million images with associated visually relevant captions.</span>

<span class="sd">    You should manually download the images using official download.m by replacing &#39;urls{i}(24, end)&#39; with</span>
<span class="sd">    &#39;urls{i}(24:1:end)&#39; and keep the directory as below.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └─ dataset_dir</span>
<span class="sd">           ├── SBU_captioned_photo_dataset_captions.txt</span>
<span class="sd">           ├── SBU_captioned_photo_dataset_urls.txt</span>
<span class="sd">           └── sbu_images</span>
<span class="sd">               ├── m_3326_3596303505_3ce4c20529.jpg</span>
<span class="sd">               ├── ......</span>
<span class="sd">               └── m_2522_4182181099_c3c23ab1cc.jpg</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @inproceedings{Ordonez:2011:im2text,</span>
<span class="sd">          Author    = {Vicente Ordonez and Girish Kulkarni and Tamara L. Berg},</span>
<span class="sd">          Title     = {Im2Text: Describing Images Using 1 Million Captioned Photographs},</span>
<span class="sd">          Booktitle = {Neural Information Processing Systems ({NIPS})},</span>
<span class="sd">          Year      = {2011},</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_sbu_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
                 <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">SBUNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span></div>


<div class="viewcode-block" id="SemeionDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.SemeionDataset.html#mindspore.dataset.SemeionDataset">[docs]</a><span class="k">class</span> <span class="nc">SemeionDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses Semeion dataset.</span>

<span class="sd">    The generated dataset has two columns :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is a scalar of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        num_samples (int, optional): The number of samples to be included in the dataset</span>
<span class="sd">            (default=None, will read all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, number set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected</span>
<span class="sd">            order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; semeion_dataset_dir = &quot;/path/to/semeion_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Get all samples from SEMEION dataset in sequence</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.SemeionDataset(dataset_dir=semeion_dataset_dir, shuffle=False)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Randomly select 10 samples from SEMEION dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.SemeionDataset(dataset_dir=semeion_dataset_dir, num_samples=10, shuffle=True)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 3) Get samples from SEMEION dataset for shard 0 in a 2-way distributed training</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.SemeionDataset(dataset_dir=semeion_dataset_dir, num_shards=2, shard_id=0)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # In SEMEION dataset, each dictionary has keys: image, label.</span>

<span class="sd">    About SEMEION dataset:</span>

<span class="sd">    The dataset was created by Tactile Srl, Brescia, Italy (http://www.tattile.it) and donated in 1994</span>
<span class="sd">    to Semeion Research Center of Sciences of Communication, Rome, Italy (http://www.semeion.it),</span>
<span class="sd">    for machine learning research.</span>

<span class="sd">    This dataset consists of 1593 records (rows) and 256 attributes (columns). Each record represents</span>
<span class="sd">    a handwritten digit, originally scanned with a resolution of 256 grey scale. Each pixel of the each</span>
<span class="sd">    original scanned image was first stretched, and after scaled between 0 and 1</span>
<span class="sd">    (setting to 0 every pixel whose value was under the value 127 of the grey scale (127 included)</span>
<span class="sd">    and setting to 1 each pixel whose original value in the grey scale was over 127). Finally, each binary image</span>
<span class="sd">    was scaled again into a 16x16 square box (the final 256 binary attributes).</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── semeion_dataset_dir</span>
<span class="sd">            └──semeion.data</span>
<span class="sd">            └──semeion.names</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @article{</span>
<span class="sd">          title={The Theory of Independent Judges, in Substance Use &amp; Misuse 33(2)1998, pp 439-461},</span>
<span class="sd">          author={M Buscema, MetaNet},</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_semeion_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">SemeionNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span></div>


<div class="viewcode-block" id="STL10Dataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.STL10Dataset.html#mindspore.dataset.STL10Dataset">[docs]</a><span class="k">class</span> <span class="nc">STL10Dataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses STL10 dataset.</span>

<span class="sd">    The generated dataset has two columns: :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is of a scalar of int32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str, optional): Usage of this dataset, can be &quot;train&quot;, &quot;test&quot;,</span>
<span class="sd">            &quot;unlabeled&quot;, &quot;train+unlabeled&quot; or &quot;all&quot; . &quot;train&quot; will read from 5,000</span>
<span class="sd">            train samples, &quot;test&quot; will read from 8,000 test samples,</span>
<span class="sd">            &quot;unlabeled&quot; will read from all 100,000 samples, and &quot;train+unlabeled&quot;</span>
<span class="sd">            will read from 105000 samples, &quot;all&quot; will read all the samples</span>
<span class="sd">            (default=None, all samples).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset.</span>
<span class="sd">            (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, number set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected</span>
<span class="sd">            order behavior shown in the table).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the</span>
<span class="sd">            dataset (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, &#39;num_samples&#39; reflects</span>
<span class="sd">            the max sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir is not valid or does not exist or does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If usage is invalid.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a sampler. &#39;sampler&#39; and &#39;shuffle&#39; are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using &#39;sampler&#39; and &#39;shuffle&#39;</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter &#39;sampler&#39;</span>
<span class="sd">         - Parameter &#39;shuffle&#39;</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; stl10_dataset_dir = &quot;/path/to/stl10_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Get all samples from STL10 dataset in sequence</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.STL10Dataset(dataset_dir=stl10_dataset_dir, shuffle=False)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Randomly select 350 samples from STL10 dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.STL10Dataset(dataset_dir=stl10_dataset_dir, num_samples=350, shuffle=True)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 3) Get samples from STL10 dataset for shard 0 in a 2-way distributed training</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.STL10Dataset(dataset_dir=stl10_dataset_dir, num_shards=2, shard_id=0)</span>

<span class="sd">    About STL10 dataset:</span>

<span class="sd">    STL10 dataset consists of 10 classes: airplane, bird, car, cat, deer, dog, horse, monkey, ship, truck.</span>
<span class="sd">    STL10 is is inspired by the CIFAR-10 dataset.</span>
<span class="sd">    Images are 96x96 pixels, color.</span>
<span class="sd">    500 training images, 800 test images per class and 100000 unlabeled images.</span>
<span class="sd">    Labels are 0-indexed, and unlabeled images have -1 as their labels.</span>

<span class="sd">    Here is the original STL10 dataset structure.</span>
<span class="sd">    You can unzip the dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── stl10_dataset_dir</span>
<span class="sd">             ├── train_X.bin</span>
<span class="sd">             ├── train_y.bin</span>
<span class="sd">             ├── test_X.bin</span>
<span class="sd">             ├── test_y.bin</span>
<span class="sd">             └── unlabeled_X.bin</span>

<span class="sd">    Citation of STL10 dataset.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @techreport{Coates10,</span>
<span class="sd">        author       = {Adam Coates},</span>
<span class="sd">        title        = {Learning multiple layers of features from tiny images},</span>
<span class="sd">        year         = {20010},</span>
<span class="sd">        howpublished = {https://cs.stanford.edu/~acoates/stl10/},</span>
<span class="sd">        description  = {The STL-10 dataset consists of 96x96 RGB images in 10 classes,</span>
<span class="sd">                        with 500 training images and 800 testing images per class.</span>
<span class="sd">                        There are 5000 training images and 8000 test images.</span>
<span class="sd">                        It also has 100000 unlabeled images for unsupervised learning.</span>
<span class="sd">                        These examples are extracted from a similar but broader distribution of images.</span>
<span class="sd">                        }</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_stl10_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">STL10Node</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span></div>


<span class="k">class</span> <span class="nc">_SVHNDataset</span><span class="p">:</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Mainly for loading SVHN Dataset, and return two rows each time.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">realpath</span><span class="p">(</span><span class="n">dataset_dir</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">usage</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">column_names</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">,</span> <span class="s2">&quot;label&quot;</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage_all</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;train&quot;</span><span class="p">,</span> <span class="s2">&quot;test&quot;</span><span class="p">,</span> <span class="s2">&quot;extra&quot;</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">uint32</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">==</span> <span class="s2">&quot;all&quot;</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">_usage</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage_all</span><span class="p">:</span>
                <span class="n">data</span><span class="p">,</span> <span class="n">label</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_load_mat</span><span class="p">(</span><span class="n">_usage</span><span class="p">)</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="n">data</span><span class="p">))</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">size</span> <span class="k">else</span> <span class="n">data</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">,</span> <span class="n">label</span><span class="p">))</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="o">.</span><span class="n">size</span> <span class="k">else</span> <span class="n">label</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_load_mat</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_load_mat</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mode</span><span class="p">):</span>
        <span class="n">filename</span> <span class="o">=</span> <span class="n">mode</span> <span class="o">+</span> <span class="s2">&quot;_32x32.mat&quot;</span>
        <span class="n">mat_data</span> <span class="o">=</span> <span class="n">loadmat</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="n">filename</span><span class="p">))</span>
        <span class="n">data</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">mat_data</span><span class="p">[</span><span class="s1">&#39;X&#39;</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span>
        <span class="n">label</span> <span class="o">=</span> <span class="n">mat_data</span><span class="p">[</span><span class="s1">&#39;y&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">uint32</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span>
        <span class="n">np</span><span class="o">.</span><span class="n">place</span><span class="p">(</span><span class="n">label</span><span class="p">,</span> <span class="n">label</span> <span class="o">==</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">label</span>

    <span class="k">def</span> <span class="fm">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">index</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">index</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">[</span><span class="n">index</span><span class="p">]</span>

    <span class="k">def</span> <span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>


<div class="viewcode-block" id="SVHNDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.SVHNDataset.html#mindspore.dataset.SVHNDataset">[docs]</a><span class="k">class</span> <span class="nc">SVHNDataset</span><span class="p">(</span><span class="n">GeneratorDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses SVHN dataset.</span>

<span class="sd">    The generated dataset has two columns: :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is of a scalar of uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str, optional): Specify the &#39;train&#39;, &#39;test&#39;, &#39;extra&#39; or &#39;all&#39; parts of dataset</span>
<span class="sd">            (default=None, will read all samples).</span>
<span class="sd">        num_samples (int, optional): The number of samples to be included in the dataset (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of subprocesses used to fetch the dataset in parallel (default=1).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset. Random accessible input is required.</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        sampler (Union[Sampler, Iterable], optional): Object used to choose samples from the dataset. Random accessible</span>
<span class="sd">            input is required (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            Random accessible input is required. When this argument is specified, &#39;num_samples&#39; reflects the max</span>
<span class="sd">            sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This argument must be specified only</span>
<span class="sd">            when num_shards is also specified. Random accessible input is required.</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir is not valid or does not exist or does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If usage is invalid.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a sampler. &#39;sampler&#39; and &#39;shuffle&#39; are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using &#39;sampler&#39; and &#39;shuffle&#39;</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter &#39;sampler&#39;</span>
<span class="sd">         - Parameter &#39;shuffle&#39;</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; svhn_dataset_dir = &quot;/path/to/svhn_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.SVHNDataset(dataset_dir=svhn_dataset_dir, usage=&quot;train&quot;)</span>

<span class="sd">    About SVHN dataset:</span>

<span class="sd">    SVHN dataset consists of 10 digit classes.</span>
<span class="sd">    SVHN is obtained from house numbers in Google Street View images.</span>
<span class="sd">    73257 digits for training, 26032 digits for testing, and 531131 additional extra training data.</span>

<span class="sd">    Here is the original SVHN dataset structure.</span>
<span class="sd">    You can unzip the dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── svhn_dataset_dir</span>
<span class="sd">             ├── train_32x32.mat</span>
<span class="sd">             ├── test_32x32.mat</span>
<span class="sd">             └── extra_32x32.mat</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @article{</span>
<span class="sd">          title={Reading Digits in Natural Images with Unsupervised Feature Learning},</span>
<span class="sd">          author={Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Bo Wu, Andrew Y. Ng},</span>
<span class="sd">          conference={NIPS Workshop on Deep Learning and Unsupervised Feature Learning 2011.},</span>
<span class="sd">          year={2011},</span>
<span class="sd">          publisher={NIPS}</span>
<span class="sd">          url={http://ufldl.stanford.edu/housenumbers}</span>
<span class="sd">        }</span>

<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_svhn_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">realpath</span><span class="p">(</span><span class="n">dataset_dir</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>
        <span class="n">dataset</span> <span class="o">=</span> <span class="n">_SVHNDataset</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">)</span>

        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">column_names</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">column_names</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span>
                         <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">)</span></div>


<div class="viewcode-block" id="USPSDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.USPSDataset.html#mindspore.dataset.USPSDataset">[docs]</a><span class="k">class</span> <span class="nc">USPSDataset</span><span class="p">(</span><span class="n">SourceDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses the USPS dataset.</span>

<span class="sd">    The generated dataset has two columns: :py:obj:`[image, label]`.</span>
<span class="sd">    The tensor of column :py:obj:`image` is of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`label` is of a scalar of uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str, optional): Usage of this dataset, can be &quot;train&quot;, &quot;test&quot; or &quot;all&quot;. &quot;train&quot; will read from 7,291</span>
<span class="sd">            train samples, &quot;test&quot; will read from 2,007 test samples, &quot;all&quot; will read from all 9,298 samples.</span>
<span class="sd">            (default=None, will read all samples)</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, will read all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, will use value set in the config).</span>
<span class="sd">        shuffle (Union[bool, Shuffle level], optional): Perform reshuffling of the data every epoch</span>
<span class="sd">            (default=Shuffle.GLOBAL).</span>
<span class="sd">            If shuffle is False, no shuffling will be performed;</span>
<span class="sd">            If shuffle is True, the behavior is the same as setting shuffle to be Shuffle.GLOBAL</span>
<span class="sd">            Otherwise, there are two levels of shuffling:</span>

<span class="sd">            - Shuffle.GLOBAL: Shuffle both the files and samples.</span>

<span class="sd">            - Shuffle.FILES: Shuffle files only.</span>

<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            When this argument is specified, `num_samples` reflects the max sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This</span>
<span class="sd">            argument can only be specified when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir is not valid or does not exist or does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If usage is invalid.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; usps_dataset_dir = &quot;/path/to/usps_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Read 3 samples from USPS dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.USPSDataset(dataset_dir=usps_dataset_dir, num_samples=3)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Note: In USPS dataset, each dictionary has keys &quot;image&quot; and &quot;label&quot;</span>

<span class="sd">    About USPS dataset:</span>

<span class="sd">    USPS is a digit dataset automatically scanned from envelopes by the U.S. Postal Service</span>
<span class="sd">    containing a total of 9,298 16×16 pixel grayscale samples.</span>
<span class="sd">    The images are centered, normalized and show a broad range of font styles.</span>

<span class="sd">    Here is the original USPS dataset structure.</span>
<span class="sd">    You can download and unzip the dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── usps_dataset_dir</span>
<span class="sd">             ├── usps</span>
<span class="sd">             ├── usps.t</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @article{hull1994database,</span>
<span class="sd">          title={A database for handwritten text recognition research},</span>
<span class="sd">          author={Hull, Jonathan J.},</span>
<span class="sd">          journal={IEEE Transactions on pattern analysis and machine intelligence},</span>
<span class="sd">          volume={16},</span>
<span class="sd">          number={5},</span>
<span class="sd">          pages={550--554},</span>
<span class="sd">          year={1994},</span>
<span class="sd">          publisher={IEEE}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_usps_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="n">Shuffle</span><span class="o">.</span><span class="n">GLOBAL</span><span class="p">,</span>
                 <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span>
                         <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">USPSNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_samples</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">shuffle_flag</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span><span class="p">,</span>
                            <span class="bp">self</span><span class="o">.</span><span class="n">shard_id</span><span class="p">)</span></div>


<span class="k">class</span> <span class="nc">VOCDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses VOC dataset.</span>

<span class="sd">    The generated dataset with different task setting has different output columns:</span>

<span class="sd">    - task = :py:obj:`Detection`, output columns: :py:obj:`[image, dtype=uint8]`, :py:obj:`[bbox, dtype=float32]`, \</span>
<span class="sd">        :py:obj:`[label, dtype=uint32]`, :py:obj:`[difficult, dtype=uint32]`, :py:obj:`[truncate, dtype=uint32]`.</span>
<span class="sd">    - task = :py:obj:`Segmentation`, output columns: :py:obj:`[image, dtype=uint8]`, :py:obj:`[target,dtype=uint8]`.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        task (str, optional): Set the task type of reading voc data, now only support `Segmentation` or `Detection`</span>
<span class="sd">            (default= `Segmentation`).</span>
<span class="sd">        usage (str, optional): Set the task type of ImageSets(default= `train`). If task is `Segmentation`, image and</span>
<span class="sd">            annotation list will be loaded in ./ImageSets/Segmentation/usage + &quot;.txt&quot;; If task is `Detection`, image and</span>
<span class="sd">            annotation list will be loaded in ./ImageSets/Main/usage + &quot;.txt&quot;; if task and usage are not set, image and</span>
<span class="sd">            annotation list will be loaded in ./ImageSets/Segmentation/train.txt as default.</span>
<span class="sd">        class_indexing (dict, optional): A str-to-int mapping from label name to index, only valid in</span>
<span class="sd">            `Detection` task (default=None, the folder names will be sorted alphabetically and each</span>
<span class="sd">            class will be given a unique index starting from 0).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, number set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None, expected</span>
<span class="sd">            order behavior shown in the table).</span>
<span class="sd">        decode (bool, optional): Decode the images after reading (default=False).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided</span>
<span class="sd">            into (default=None). When this argument is specified, `num_samples` reflects</span>
<span class="sd">            the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within num_shards (default=None). This</span>
<span class="sd">            argument can only be specified when num_shards is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing.</span>
<span class="sd">            (default=None, which means no cache is used).</span>
<span class="sd">        extra_metadata(bool, optional): Flag to add extra meta-data to row. If True, an additional column named</span>
<span class="sd">            :py:obj:`[_meta-filename, dtype=string]` will be output at the end (default=False).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If xml of Annotations is an invalid format.</span>
<span class="sd">        RuntimeError: If xml of Annotations loss attribution of `object`.</span>
<span class="sd">        RuntimeError: If xml of Annotations loss attribution of `bndbox`.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If task is not equal &#39;Segmentation&#39; or &#39;Detection&#39;.</span>
<span class="sd">        ValueError: If task equal &#39;Segmentation&#39; but class_indexing is not None.</span>
<span class="sd">        ValueError: If txt related to mode is not exist.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - Column &#39;[_meta-filename, dtype=string]&#39; won&#39;t be output unless an explicit rename dataset op</span>
<span class="sd">          is added to remove the prefix(&#39;_meta-&#39;).</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; voc_dataset_dir = &quot;/path/to/voc_dataset_directory&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 1) Read VOC data for segmentation training</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.VOCDataset(dataset_dir=voc_dataset_dir, task=&quot;Segmentation&quot;, usage=&quot;train&quot;)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 2) Read VOC data for detection training</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.VOCDataset(dataset_dir=voc_dataset_dir, task=&quot;Detection&quot;, usage=&quot;train&quot;)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 3) Read all VOC dataset samples in voc_dataset_dir with 8 threads in random order</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.VOCDataset(dataset_dir=voc_dataset_dir, task=&quot;Detection&quot;, usage=&quot;train&quot;,</span>
<span class="sd">        ...                         num_parallel_workers=8)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # 4) Read then decode all VOC dataset samples in voc_dataset_dir in sequence</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.VOCDataset(dataset_dir=voc_dataset_dir, task=&quot;Detection&quot;, usage=&quot;train&quot;,</span>
<span class="sd">        ...                         decode=True, shuffle=False)</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # In VOC dataset, if task=&#39;Segmentation&#39;, each dictionary has keys &quot;image&quot; and &quot;target&quot;</span>
<span class="sd">        &gt;&gt;&gt; # In VOC dataset, if task=&#39;Detection&#39;, each dictionary has keys &quot;image&quot; and &quot;annotation&quot;</span>

<span class="sd">    About VOC dataset.</span>

<span class="sd">    The PASCAL Visual Object Classes (VOC) challenge is a benchmark in visual</span>
<span class="sd">    object category recognition and detection, providing the vision and machine</span>
<span class="sd">    learning communities with a standard dataset of images and annotation, and</span>
<span class="sd">    standard evaluation procedures.</span>

<span class="sd">    You can unzip the original VOC-2012 dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── voc2012_dataset_dir</span>
<span class="sd">            ├── Annotations</span>
<span class="sd">            │    ├── 2007_000027.xml</span>
<span class="sd">            │    ├── 2007_000032.xml</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            ├── ImageSets</span>
<span class="sd">            │    ├── Action</span>
<span class="sd">            │    ├── Layout</span>
<span class="sd">            │    ├── Main</span>
<span class="sd">            │    └── Segmentation</span>
<span class="sd">            ├── JPEGImages</span>
<span class="sd">            │    ├── 2007_000027.jpg</span>
<span class="sd">            │    ├── 2007_000032.jpg</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            ├── SegmentationClass</span>
<span class="sd">            │    ├── 2007_000032.png</span>
<span class="sd">            │    ├── 2007_000033.png</span>
<span class="sd">            │    ├── ...</span>
<span class="sd">            └── SegmentationObject</span>
<span class="sd">                 ├── 2007_000032.png</span>
<span class="sd">                 ├── 2007_000033.png</span>
<span class="sd">                 ├── ...</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @article{Everingham10,</span>
<span class="sd">        author       = {Everingham, M. and Van~Gool, L. and Williams, C. K. I. and Winn, J. and Zisserman, A.},</span>
<span class="sd">        title        = {The Pascal Visual Object Classes (VOC) Challenge},</span>
<span class="sd">        journal      = {International Journal of Computer Vision},</span>
<span class="sd">        volume       = {88},</span>
<span class="sd">        year         = {2012},</span>
<span class="sd">        number       = {2},</span>
<span class="sd">        month        = {jun},</span>
<span class="sd">        pages        = {303--338},</span>
<span class="sd">        biburl       = {http://host.robots.ox.ac.uk/pascal/VOC/pubs/everingham10.html#bibtex},</span>
<span class="sd">        howpublished = {http://host.robots.ox.ac.uk/pascal/VOC/voc2012/index.html}</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_vocdataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">task</span><span class="o">=</span><span class="s2">&quot;Segmentation&quot;</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="s2">&quot;train&quot;</span><span class="p">,</span> <span class="n">class_indexing</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">decode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">extra_metadata</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="s2">&quot;Segmentation&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;train&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">class_indexing</span><span class="p">,</span> <span class="p">{})</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">extra_metadata</span> <span class="o">=</span> <span class="n">extra_metadata</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">VOCNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">,</span>
                           <span class="bp">self</span><span class="o">.</span><span class="n">extra_metadata</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">get_class_indexing</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Get the class index.</span>

<span class="sd">        Returns:</span>
<span class="sd">            dict, a str-to-int mapping from label name to index.</span>

<span class="sd">        Examples:</span>
<span class="sd">            &gt;&gt;&gt; voc_dataset_dir = &quot;/path/to/voc_dataset_directory&quot;</span>
<span class="sd">            &gt;&gt;&gt;</span>
<span class="sd">            &gt;&gt;&gt; dataset = ds.VOCDataset(dataset_dir=voc_dataset_dir, task=&quot;Detection&quot;)</span>
<span class="sd">            &gt;&gt;&gt; class_indexing = dataset.get_class_indexing()</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">!=</span> <span class="s2">&quot;Detection&quot;</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">&quot;Only &#39;Detection&#39; support get_class_indexing.&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_class_indexing</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">runtime_getter</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_init_tree_getters</span><span class="p">()</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_class_indexing</span> <span class="o">=</span> <span class="n">runtime_getter</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">GetClassIndexing</span><span class="p">()</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span> <span class="o">=</span> <span class="p">{}</span>
            <span class="k">for</span> <span class="n">pair</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_class_indexing</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span><span class="p">[</span><span class="n">pair</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span> <span class="o">=</span> <span class="n">pair</span><span class="p">[</span><span class="mi">1</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">class_indexing</span>


<div class="viewcode-block" id="WIDERFaceDataset"><a class="viewcode-back" href="../../../../api_python/dataset/mindspore.dataset.WIDERFaceDataset.html#mindspore.dataset.WIDERFaceDataset">[docs]</a><span class="k">class</span> <span class="nc">WIDERFaceDataset</span><span class="p">(</span><span class="n">MappableDataset</span><span class="p">,</span> <span class="n">VisionBaseDataset</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    A source dataset that reads and parses WIDERFace dataset.</span>

<span class="sd">    When usage is &quot;train&quot;, &quot;valid&quot; or &quot;all&quot;, the generated dataset has eight columns [&quot;image&quot;, &quot;bbox&quot;, &quot;blur&quot;,</span>
<span class="sd">    &quot;expression&quot;, &quot;illumination&quot;, &quot;occlusion&quot;, &quot;pose&quot;, &quot;invalid&quot;]. When usage is &quot;test&quot;, it only has one column</span>
<span class="sd">    [&quot;image&quot;].</span>
<span class="sd">    The tensor of column :py:obj:`image` is a vector of the uint8 type.</span>
<span class="sd">    The tensor of column :py:obj:`bbox` is a scalar of the uint32 type.</span>
<span class="sd">    The tensor of column :py:obj:`blur` is a scalar of the uint32 type.</span>
<span class="sd">    The tensor of column :py:obj:`expression` is a scalar of the uint32 type.</span>
<span class="sd">    The tensor of column :py:obj:`illumination` is a scalar of the uint32 type.</span>
<span class="sd">    The tensor of column :py:obj:`occlusion` is a scalar of the uint32 type.</span>
<span class="sd">    The tensor of column :py:obj:`pose` is a scalar of the uint32 type.</span>
<span class="sd">    The tensor of column :py:obj:`invalid` is a scalar of the uint32 type.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset_dir (str): Path to the root directory that contains the dataset.</span>
<span class="sd">        usage (str, optional): Usage of this dataset, can be `train`, `test`, `valid` or `all`. `train` will read</span>
<span class="sd">            from 12,880 samples, `test` will read from 16,097 samples, `valid` will read from 3,226 test samples</span>
<span class="sd">            and `all` will read all `train` and `valid` samples (default=None, will be set to `all`).</span>
<span class="sd">        num_samples (int, optional): The number of images to be included in the dataset</span>
<span class="sd">            (default=None, will read all images).</span>
<span class="sd">        num_parallel_workers (int, optional): Number of workers to read the data</span>
<span class="sd">            (default=None, will use value set in the config).</span>
<span class="sd">        shuffle (bool, optional): Whether or not to perform shuffle on the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        decode (bool, optional): Decode the images after reading (default=False).</span>
<span class="sd">        sampler (Sampler, optional): Object used to choose samples from the dataset</span>
<span class="sd">            (default=None, expected order behavior shown in the table).</span>
<span class="sd">        num_shards (int, optional): Number of shards that the dataset will be divided into (default=None).</span>
<span class="sd">            When this argument is specified, `num_samples` reflects the maximum sample number of per shard.</span>
<span class="sd">        shard_id (int, optional): The shard ID within `num_shards` (default=None). This argument can only be specified</span>
<span class="sd">            when `num_shards` is also specified.</span>
<span class="sd">        cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing</span>
<span class="sd">            (default=None, which means no cache is used).</span>

<span class="sd">    Raises:</span>
<span class="sd">        RuntimeError: If dataset_dir does not contain data files.</span>
<span class="sd">        RuntimeError: If num_parallel_workers exceeds the max thread numbers.</span>
<span class="sd">        RuntimeError: If sampler and shuffle are specified at the same time.</span>
<span class="sd">        RuntimeError: If sampler and sharding are specified at the same time.</span>
<span class="sd">        RuntimeError: If num_shards is specified but shard_id is None.</span>
<span class="sd">        RuntimeError: If shard_id is specified but num_shards is None.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>
<span class="sd">        ValueError: If usage is not in [`train`, `test`, `valid`, `all`].</span>
<span class="sd">        ValueError: If annotation_file is not exist.</span>
<span class="sd">        ValueError: If dataset_dir is not exist.</span>
<span class="sd">        ValueError: If shard_id is invalid (&lt; 0 or &gt;= num_shards).</span>

<span class="sd">    Note:</span>
<span class="sd">        - This dataset can take in a `sampler`. `sampler` and `shuffle` are mutually exclusive.</span>
<span class="sd">          The table below shows what input arguments are allowed and their expected behavior.</span>

<span class="sd">    .. list-table:: Expected Order Behavior of Using `sampler` and `shuffle`</span>
<span class="sd">       :widths: 25 25 50</span>
<span class="sd">       :header-rows: 1</span>

<span class="sd">       * - Parameter `sampler`</span>
<span class="sd">         - Parameter `shuffle`</span>
<span class="sd">         - Expected Order Behavior</span>
<span class="sd">       * - None</span>
<span class="sd">         - None</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - True</span>
<span class="sd">         - random order</span>
<span class="sd">       * - None</span>
<span class="sd">         - False</span>
<span class="sd">         - sequential order</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - None</span>
<span class="sd">         - order defined by sampler</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - True</span>
<span class="sd">         - not allowed</span>
<span class="sd">       * - Sampler object</span>
<span class="sd">         - False</span>
<span class="sd">         - not allowed</span>

<span class="sd">    Examples:</span>
<span class="sd">        &gt;&gt;&gt; wider_face_dir = &quot;/path/to/wider_face_dataset&quot;</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; # Read 3 samples from WIDERFace dataset</span>
<span class="sd">        &gt;&gt;&gt; dataset = ds.WIDERFaceDataset(dataset_dir=wider_face_dir, num_samples=3)</span>

<span class="sd">    About WIDERFace dataset:</span>

<span class="sd">    The WIDERFace database of people faces has a training set of 12,880 samples, a testing set of 16,097 examples</span>
<span class="sd">    and a validating set of 3,226 examples. It is a subset of a larger set available from WIDER. The digits have</span>
<span class="sd">    been size-normalized and centered in a fixed-size image.</span>

<span class="sd">    The following is the original WIDERFace dataset structure.</span>
<span class="sd">    You can unzip the dataset files into this directory structure and read by MindSpore&#39;s API.</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        .</span>
<span class="sd">        └── wider_face_dir</span>
<span class="sd">             ├── WIDER_test</span>
<span class="sd">             │    └── images</span>
<span class="sd">             │         ├── 0--Parade</span>
<span class="sd">             │         │     ├── 0_Parade_marchingband_1_9.jpg</span>
<span class="sd">             │         │     ├── ...</span>
<span class="sd">             │         ├──1--Handshaking</span>
<span class="sd">             │         ├──...</span>
<span class="sd">             ├── WIDER_train</span>
<span class="sd">             │    └── images</span>
<span class="sd">             │         ├── 0--Parade</span>
<span class="sd">             │         │     ├── 0_Parade_marchingband_1_11.jpg</span>
<span class="sd">             │         │     ├── ...</span>
<span class="sd">             │         ├──1--Handshaking</span>
<span class="sd">             │         ├──...</span>
<span class="sd">             ├── WIDER_val</span>
<span class="sd">             │    └── images</span>
<span class="sd">             │         ├── 0--Parade</span>
<span class="sd">             │         │     ├── 0_Parade_marchingband_1_102.jpg</span>
<span class="sd">             │         │     ├── ...</span>
<span class="sd">             │         ├──1--Handshaking</span>
<span class="sd">             │         ├──...</span>
<span class="sd">             └── wider_face_split</span>
<span class="sd">                  ├── wider_face_test_filelist.txt</span>
<span class="sd">                  ├── wider_face_train_bbx_gt.txt</span>
<span class="sd">                  └── wider_face_val_bbx_gt.txt</span>

<span class="sd">    Citation:</span>

<span class="sd">    .. code-block::</span>

<span class="sd">        @inproceedings{2016WIDER,</span>
<span class="sd">          title={WIDER FACE: A Face Detection Benchmark},</span>
<span class="sd">          author={Yang, S. and Luo, P. and Loy, C. C. and Tang, X.},</span>
<span class="sd">          booktitle={IEEE},</span>
<span class="sd">          pages={5525-5533},</span>
<span class="sd">          year={2016},</span>
<span class="sd">        }</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@check_wider_face_dataset</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_parallel_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">decode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">num_parallel_workers</span><span class="o">=</span><span class="n">num_parallel_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
                         <span class="n">shuffle</span><span class="o">=</span><span class="n">shuffle</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">shard_id</span><span class="o">=</span><span class="n">shard_id</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span> <span class="o">=</span> <span class="n">dataset_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">usage</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">usage</span><span class="p">,</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">decode</span> <span class="o">=</span> <span class="n">replace_none</span><span class="p">(</span><span class="n">decode</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">children</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">cde</span><span class="o">.</span><span class="n">WIDERFaceNode</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">usage</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">)</span></div>
</pre></div>

           </div>
           
          </div>
          <footer>

  <hr/>

  <div role="contentinfo">
    <p>
        &#169; Copyright 2021, MindSpore.

    </p>
  </div>
    
    
    
    Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
    
    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
    
    provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>
        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
   

</body>
</html>